From 82aee2c3763b34146920b9eab2b79278b8873963 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 12 Jun 2024 15:05:36 +0800 Subject: [PATCH 1/9] fixup --- .../src/main/scala/org/apache/gluten/GlutenConfig.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index a4e5a4425e3b..121e3f332c7d 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -28,7 +28,6 @@ import java.util.Locale import java.util.concurrent.TimeUnit import scala.collection.JavaConverters._ -import scala.collection.JavaConverters.collectionAsScalaIterableConverter case class GlutenNumaBindingInfo( enableNumaBinding: Boolean, @@ -307,10 +306,12 @@ class GlutenConfig(conf: SQLConf) extends Logging { def veloxCoalesceBatchesBeforeShuffle: Boolean = conf.getConf(COLUMNAR_VELOX_COALESCE_BATCHES_BEFORE_SHUFFLE) - def veloxMinBatchSizeForShuffle: Int = + def veloxMinBatchSizeForShuffle: Int = { + val defaultSize: Int = (0.8 * conf.getConf(COLUMNAR_MAX_BATCH_SIZE)).toInt.max(1) conf .getConf(COLUMNAR_VELOX_MIN_BATCH_SIZE_FOR_SHUFFLE) - .getOrElse(conf.getConf(COLUMNAR_MAX_BATCH_SIZE)) + .getOrElse(defaultSize) + } def chColumnarShufflePreferSpill: Boolean = conf.getConf(COLUMNAR_CH_SHUFFLE_PREFER_SPILL_ENABLED) @@ -1425,7 +1426,7 @@ object GlutenConfig { buildConf("spark.gluten.sql.columnar.backend.velox.coalesceBatchesBeforeShuffle") .internal() .doc(s"If true, combine small columnar batches together before sending to shuffle. " + - s"The default minimum output batch size is equal to $GLUTEN_MAX_BATCH_SIZE_KEY") + s"The default minimum output batch size is equal to 0.8 * $GLUTEN_MAX_BATCH_SIZE_KEY") .booleanConf .createWithDefault(false) From 99dc0965af3aad642e25c1f4040c528aec726d64 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 12 Jun 2024 15:06:30 +0800 Subject: [PATCH 2/9] fixup --- .../common/src/main/scala/org/apache/gluten/GlutenConfig.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index 121e3f332c7d..eb6118ffc74e 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -1428,7 +1428,7 @@ object GlutenConfig { .doc(s"If true, combine small columnar batches together before sending to shuffle. " + s"The default minimum output batch size is equal to 0.8 * $GLUTEN_MAX_BATCH_SIZE_KEY") .booleanConf - .createWithDefault(false) + .createWithDefault(true) val COLUMNAR_VELOX_MIN_BATCH_SIZE_FOR_SHUFFLE = buildConf("spark.gluten.sql.columnar.backend.velox.minBatchSizeForShuffle") From 481a4964b5450916021e0ddbc553bd3f0aa98370 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Fri, 14 Jun 2024 13:55:38 +0800 Subject: [PATCH 3/9] fixup --- .../FlushableHashAggregateRule.scala | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala b/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala index f850b6f457ea..c4b9a768fa84 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension -import org.apache.gluten.execution.{FlushableHashAggregateExecTransformer, HashAggregateExecTransformer, ProjectExecTransformer, RegularHashAggregateExecTransformer} +import org.apache.gluten.execution._ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.aggregate.{Partial, PartialMerge} @@ -30,11 +30,12 @@ import org.apache.spark.sql.execution.exchange.ShuffleExchangeLike * optimizations such as flushing and abandoning. */ case class FlushableHashAggregateRule(session: SparkSession) extends Rule[SparkPlan] { + import FlushableHashAggregateRule._ override def apply(plan: SparkPlan): SparkPlan = plan.transformUp { - case shuffle: ShuffleExchangeLike => + case ShuffleAndChild(shuffle, child) => // If an exchange follows a hash aggregate in which all functions are in partial mode, // then it's safe to convert the hash aggregate to flushable hash aggregate. - shuffle.child match { + child match { case HashAggPropagatedToShuffle(proj, agg) => shuffle.withNewChildren( Seq(proj.withNewChildren(Seq(FlushableHashAggregateExecTransformer( @@ -125,4 +126,16 @@ object FlushableHashAggregateRule { val distribution = ClusteredDistribution(agg.groupingExpressions) agg.child.outputPartitioning.satisfies(distribution) } + + private object ShuffleAndChild { + def unapply(plan: SparkPlan): Option[(SparkPlan, SparkPlan)] = plan match { + case s: ShuffleExchangeLike => + val child = s.child match { + case VeloxAppendBatchesExec(child, _) => child + case other => other + } + Some(s, child) + case other => None + } + } } From 89a8dfe92cc41dc771fd7ebe95b3c877d12de662 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Fri, 14 Jun 2024 14:04:17 +0800 Subject: [PATCH 4/9] fixup --- .../v1-bhj-ras/spark32/1.txt | 83 +- .../v1-bhj-ras/spark32/10.txt | 189 ++--- .../v1-bhj-ras/spark32/11.txt | 157 ++-- .../v1-bhj-ras/spark32/12.txt | 117 +-- .../v1-bhj-ras/spark32/13.txt | 137 ++-- .../v1-bhj-ras/spark32/14.txt | 105 +-- .../v1-bhj-ras/spark32/15.txt | 123 +-- .../v1-bhj-ras/spark32/16.txt | 155 ++-- .../v1-bhj-ras/spark32/17.txt | 127 ++-- .../v1-bhj-ras/spark32/18.txt | 306 ++++---- .../v1-bhj-ras/spark32/19.txt | 101 +-- .../v1-bhj-ras/spark32/20.txt | 398 +++++----- .../v1-bhj-ras/spark32/21.txt | 247 +++--- .../v1-bhj-ras/spark32/22.txt | 105 +-- .../v1-bhj-ras/spark32/3.txt | 155 ++-- .../v1-bhj-ras/spark32/4.txt | 123 +-- .../v1-bhj-ras/spark32/5.txt | 259 +++---- .../v1-bhj-ras/spark32/6.txt | 67 +- .../v1-bhj-ras/spark32/7.txt | 243 +++--- .../v1-bhj-ras/spark32/8.txt | 329 ++++---- .../v1-bhj-ras/spark32/9.txt | 253 ++++--- .../v1-bhj-ras/spark33/1.txt | 83 +- .../v1-bhj-ras/spark33/10.txt | 189 ++--- .../v1-bhj-ras/spark33/11.txt | 315 ++++---- .../v1-bhj-ras/spark33/12.txt | 117 +-- .../v1-bhj-ras/spark33/13.txt | 137 ++-- .../v1-bhj-ras/spark33/14.txt | 105 +-- .../v1-bhj-ras/spark33/15.txt | 209 +++--- .../v1-bhj-ras/spark33/16.txt | 155 ++-- .../v1-bhj-ras/spark33/17.txt | 127 ++-- .../v1-bhj-ras/spark33/18.txt | 306 ++++---- .../v1-bhj-ras/spark33/19.txt | 101 +-- .../v1-bhj-ras/spark33/20.txt | 390 +++++----- .../v1-bhj-ras/spark33/21.txt | 243 +++--- .../v1-bhj-ras/spark33/22.txt | 175 ++--- .../v1-bhj-ras/spark33/3.txt | 155 ++-- .../v1-bhj-ras/spark33/4.txt | 123 +-- .../v1-bhj-ras/spark33/5.txt | 259 +++---- .../v1-bhj-ras/spark33/6.txt | 67 +- .../v1-bhj-ras/spark33/7.txt | 243 +++--- .../v1-bhj-ras/spark33/8.txt | 329 ++++---- .../v1-bhj-ras/spark33/9.txt | 253 ++++--- .../v1-bhj-ras/spark34/1.txt | 83 +- .../v1-bhj-ras/spark34/10.txt | 189 ++--- .../v1-bhj-ras/spark34/11.txt | 315 ++++---- .../v1-bhj-ras/spark34/12.txt | 117 +-- .../v1-bhj-ras/spark34/13.txt | 137 ++-- .../v1-bhj-ras/spark34/14.txt | 105 +-- .../v1-bhj-ras/spark34/15.txt | 209 +++--- .../v1-bhj-ras/spark34/16.txt | 155 ++-- .../v1-bhj-ras/spark34/17.txt | 127 ++-- .../v1-bhj-ras/spark34/18.txt | 306 ++++---- .../v1-bhj-ras/spark34/19.txt | 101 +-- .../v1-bhj-ras/spark34/20.txt | 253 ++++--- .../v1-bhj-ras/spark34/21.txt | 243 +++--- .../v1-bhj-ras/spark34/22.txt | 175 ++--- .../v1-bhj-ras/spark34/3.txt | 155 ++-- .../v1-bhj-ras/spark34/4.txt | 123 +-- .../v1-bhj-ras/spark34/5.txt | 259 +++---- .../v1-bhj-ras/spark34/6.txt | 67 +- .../v1-bhj-ras/spark34/7.txt | 243 +++--- .../v1-bhj-ras/spark34/8.txt | 329 ++++---- .../v1-bhj-ras/spark34/9.txt | 253 ++++--- .../tpch-approved-plan/v1-bhj/spark32/1.txt | 83 +- .../tpch-approved-plan/v1-bhj/spark32/10.txt | 189 ++--- .../tpch-approved-plan/v1-bhj/spark32/11.txt | 157 ++-- .../tpch-approved-plan/v1-bhj/spark32/12.txt | 117 +-- .../tpch-approved-plan/v1-bhj/spark32/13.txt | 137 ++-- .../tpch-approved-plan/v1-bhj/spark32/14.txt | 105 +-- .../tpch-approved-plan/v1-bhj/spark32/15.txt | 123 +-- .../tpch-approved-plan/v1-bhj/spark32/16.txt | 155 ++-- .../tpch-approved-plan/v1-bhj/spark32/17.txt | 127 ++-- .../tpch-approved-plan/v1-bhj/spark32/18.txt | 306 ++++---- .../tpch-approved-plan/v1-bhj/spark32/19.txt | 101 +-- .../tpch-approved-plan/v1-bhj/spark32/20.txt | 398 +++++----- .../tpch-approved-plan/v1-bhj/spark32/21.txt | 247 +++--- .../tpch-approved-plan/v1-bhj/spark32/22.txt | 105 +-- .../tpch-approved-plan/v1-bhj/spark32/3.txt | 155 ++-- .../tpch-approved-plan/v1-bhj/spark32/4.txt | 123 +-- .../tpch-approved-plan/v1-bhj/spark32/5.txt | 259 +++---- .../tpch-approved-plan/v1-bhj/spark32/6.txt | 67 +- .../tpch-approved-plan/v1-bhj/spark32/7.txt | 243 +++--- .../tpch-approved-plan/v1-bhj/spark32/8.txt | 329 ++++---- .../tpch-approved-plan/v1-bhj/spark32/9.txt | 253 ++++--- .../tpch-approved-plan/v1-bhj/spark33/1.txt | 83 +- .../tpch-approved-plan/v1-bhj/spark33/10.txt | 189 ++--- .../tpch-approved-plan/v1-bhj/spark33/11.txt | 315 ++++---- .../tpch-approved-plan/v1-bhj/spark33/12.txt | 117 +-- .../tpch-approved-plan/v1-bhj/spark33/13.txt | 137 ++-- .../tpch-approved-plan/v1-bhj/spark33/14.txt | 105 +-- .../tpch-approved-plan/v1-bhj/spark33/15.txt | 209 +++--- .../tpch-approved-plan/v1-bhj/spark33/16.txt | 155 ++-- .../tpch-approved-plan/v1-bhj/spark33/17.txt | 127 ++-- .../tpch-approved-plan/v1-bhj/spark33/18.txt | 306 ++++---- .../tpch-approved-plan/v1-bhj/spark33/19.txt | 101 +-- .../tpch-approved-plan/v1-bhj/spark33/20.txt | 390 +++++----- .../tpch-approved-plan/v1-bhj/spark33/21.txt | 243 +++--- .../tpch-approved-plan/v1-bhj/spark33/22.txt | 207 ++--- .../tpch-approved-plan/v1-bhj/spark33/3.txt | 155 ++-- .../tpch-approved-plan/v1-bhj/spark33/4.txt | 123 +-- .../tpch-approved-plan/v1-bhj/spark33/5.txt | 259 +++---- .../tpch-approved-plan/v1-bhj/spark33/6.txt | 67 +- .../tpch-approved-plan/v1-bhj/spark33/7.txt | 243 +++--- .../tpch-approved-plan/v1-bhj/spark33/8.txt | 329 ++++---- .../tpch-approved-plan/v1-bhj/spark33/9.txt | 253 ++++--- .../tpch-approved-plan/v1-bhj/spark34/1.txt | 83 +- .../tpch-approved-plan/v1-bhj/spark34/10.txt | 189 ++--- .../tpch-approved-plan/v1-bhj/spark34/11.txt | 315 ++++---- .../tpch-approved-plan/v1-bhj/spark34/12.txt | 117 +-- .../tpch-approved-plan/v1-bhj/spark34/13.txt | 137 ++-- .../tpch-approved-plan/v1-bhj/spark34/14.txt | 105 +-- .../tpch-approved-plan/v1-bhj/spark34/15.txt | 209 +++--- .../tpch-approved-plan/v1-bhj/spark34/16.txt | 155 ++-- .../tpch-approved-plan/v1-bhj/spark34/17.txt | 127 ++-- .../tpch-approved-plan/v1-bhj/spark34/18.txt | 306 ++++---- .../tpch-approved-plan/v1-bhj/spark34/19.txt | 101 +-- .../tpch-approved-plan/v1-bhj/spark34/20.txt | 253 ++++--- .../tpch-approved-plan/v1-bhj/spark34/21.txt | 243 +++--- .../tpch-approved-plan/v1-bhj/spark34/22.txt | 207 ++--- .../tpch-approved-plan/v1-bhj/spark34/3.txt | 155 ++-- .../tpch-approved-plan/v1-bhj/spark34/4.txt | 123 +-- .../tpch-approved-plan/v1-bhj/spark34/5.txt | 259 +++---- .../tpch-approved-plan/v1-bhj/spark34/6.txt | 67 +- .../tpch-approved-plan/v1-bhj/spark34/7.txt | 243 +++--- .../tpch-approved-plan/v1-bhj/spark34/8.txt | 329 ++++---- .../tpch-approved-plan/v1-bhj/spark34/9.txt | 253 ++++--- .../tpch-approved-plan/v1-ras/spark32/1.txt | 83 +- .../tpch-approved-plan/v1-ras/spark32/10.txt | 340 +++++---- .../tpch-approved-plan/v1-ras/spark32/11.txt | 279 +++---- .../tpch-approved-plan/v1-ras/spark32/12.txt | 185 ++--- .../tpch-approved-plan/v1-ras/spark32/13.txt | 199 ++--- .../tpch-approved-plan/v1-ras/spark32/14.txt | 132 ++-- .../tpch-approved-plan/v1-ras/spark32/15.txt | 168 +++-- .../tpch-approved-plan/v1-ras/spark32/16.txt | 241 +++--- .../tpch-approved-plan/v1-ras/spark32/17.txt | 216 +++--- .../tpch-approved-plan/v1-ras/spark32/18.txt | 377 +++++----- .../tpch-approved-plan/v1-ras/spark32/19.txt | 128 ++-- .../tpch-approved-plan/v1-ras/spark32/20.txt | 502 +++++++------ .../tpch-approved-plan/v1-ras/spark32/21.txt | 471 ++++++------ .../tpch-approved-plan/v1-ras/spark32/22.txt | 173 +++-- .../tpch-approved-plan/v1-ras/spark32/3.txt | 230 +++--- .../tpch-approved-plan/v1-ras/spark32/4.txt | 189 ++--- .../tpch-approved-plan/v1-ras/spark32/5.txt | 533 +++++++------ .../tpch-approved-plan/v1-ras/spark32/6.txt | 67 +- .../tpch-approved-plan/v1-ras/spark32/7.txt | 506 +++++++------ .../tpch-approved-plan/v1-ras/spark32/8.txt | 709 ++++++++++-------- .../tpch-approved-plan/v1-ras/spark32/9.txt | 529 +++++++------ .../tpch-approved-plan/v1-ras/spark33/1.txt | 83 +- .../tpch-approved-plan/v1-ras/spark33/10.txt | 340 +++++---- .../tpch-approved-plan/v1-ras/spark33/11.txt | 473 ++++++------ .../tpch-approved-plan/v1-ras/spark33/12.txt | 185 ++--- .../tpch-approved-plan/v1-ras/spark33/13.txt | 199 ++--- .../tpch-approved-plan/v1-ras/spark33/14.txt | 132 ++-- .../tpch-approved-plan/v1-ras/spark33/15.txt | 256 ++++--- .../tpch-approved-plan/v1-ras/spark33/16.txt | 241 +++--- .../tpch-approved-plan/v1-ras/spark33/17.txt | 216 +++--- .../tpch-approved-plan/v1-ras/spark33/18.txt | 377 +++++----- .../tpch-approved-plan/v1-ras/spark33/19.txt | 128 ++-- .../tpch-approved-plan/v1-ras/spark33/20.txt | 494 ++++++------ .../tpch-approved-plan/v1-ras/spark33/21.txt | 467 ++++++------ .../tpch-approved-plan/v1-ras/spark33/22.txt | 243 +++--- .../tpch-approved-plan/v1-ras/spark33/3.txt | 230 +++--- .../tpch-approved-plan/v1-ras/spark33/4.txt | 189 ++--- .../tpch-approved-plan/v1-ras/spark33/5.txt | 533 +++++++------ .../tpch-approved-plan/v1-ras/spark33/6.txt | 67 +- .../tpch-approved-plan/v1-ras/spark33/7.txt | 506 +++++++------ .../tpch-approved-plan/v1-ras/spark33/8.txt | 709 ++++++++++-------- .../tpch-approved-plan/v1-ras/spark33/9.txt | 529 +++++++------ .../tpch-approved-plan/v1-ras/spark34/1.txt | 83 +- .../tpch-approved-plan/v1-ras/spark34/10.txt | 340 +++++---- .../tpch-approved-plan/v1-ras/spark34/11.txt | 473 ++++++------ .../tpch-approved-plan/v1-ras/spark34/12.txt | 185 ++--- .../tpch-approved-plan/v1-ras/spark34/13.txt | 199 ++--- .../tpch-approved-plan/v1-ras/spark34/14.txt | 132 ++-- .../tpch-approved-plan/v1-ras/spark34/15.txt | 256 ++++--- .../tpch-approved-plan/v1-ras/spark34/16.txt | 241 +++--- .../tpch-approved-plan/v1-ras/spark34/17.txt | 216 +++--- .../tpch-approved-plan/v1-ras/spark34/18.txt | 377 +++++----- .../tpch-approved-plan/v1-ras/spark34/19.txt | 128 ++-- .../tpch-approved-plan/v1-ras/spark34/20.txt | 494 ++++++------ .../tpch-approved-plan/v1-ras/spark34/21.txt | 467 ++++++------ .../tpch-approved-plan/v1-ras/spark34/22.txt | 243 +++--- .../tpch-approved-plan/v1-ras/spark34/3.txt | 230 +++--- .../tpch-approved-plan/v1-ras/spark34/4.txt | 189 ++--- .../tpch-approved-plan/v1-ras/spark34/5.txt | 533 +++++++------ .../tpch-approved-plan/v1-ras/spark34/6.txt | 67 +- .../tpch-approved-plan/v1-ras/spark34/7.txt | 506 +++++++------ .../tpch-approved-plan/v1-ras/spark34/8.txt | 709 ++++++++++-------- .../tpch-approved-plan/v1-ras/spark34/9.txt | 529 +++++++------ .../tpch-approved-plan/v1/spark32/1.txt | 83 +- .../tpch-approved-plan/v1/spark32/10.txt | 340 +++++---- .../tpch-approved-plan/v1/spark32/11.txt | 279 +++---- .../tpch-approved-plan/v1/spark32/12.txt | 185 ++--- .../tpch-approved-plan/v1/spark32/13.txt | 199 ++--- .../tpch-approved-plan/v1/spark32/14.txt | 132 ++-- .../tpch-approved-plan/v1/spark32/15.txt | 168 +++-- .../tpch-approved-plan/v1/spark32/16.txt | 241 +++--- .../tpch-approved-plan/v1/spark32/17.txt | 216 +++--- .../tpch-approved-plan/v1/spark32/18.txt | 377 +++++----- .../tpch-approved-plan/v1/spark32/19.txt | 128 ++-- .../tpch-approved-plan/v1/spark32/20.txt | 502 +++++++------ .../tpch-approved-plan/v1/spark32/21.txt | 471 ++++++------ .../tpch-approved-plan/v1/spark32/22.txt | 173 +++-- .../tpch-approved-plan/v1/spark32/3.txt | 230 +++--- .../tpch-approved-plan/v1/spark32/4.txt | 189 ++--- .../tpch-approved-plan/v1/spark32/5.txt | 533 +++++++------ .../tpch-approved-plan/v1/spark32/6.txt | 67 +- .../tpch-approved-plan/v1/spark32/7.txt | 506 +++++++------ .../tpch-approved-plan/v1/spark32/8.txt | 709 ++++++++++-------- .../tpch-approved-plan/v1/spark32/9.txt | 529 +++++++------ .../tpch-approved-plan/v1/spark33/1.txt | 83 +- .../tpch-approved-plan/v1/spark33/10.txt | 340 +++++---- .../tpch-approved-plan/v1/spark33/11.txt | 473 ++++++------ .../tpch-approved-plan/v1/spark33/12.txt | 185 ++--- .../tpch-approved-plan/v1/spark33/13.txt | 199 ++--- .../tpch-approved-plan/v1/spark33/14.txt | 132 ++-- .../tpch-approved-plan/v1/spark33/15.txt | 256 ++++--- .../tpch-approved-plan/v1/spark33/16.txt | 241 +++--- .../tpch-approved-plan/v1/spark33/17.txt | 216 +++--- .../tpch-approved-plan/v1/spark33/18.txt | 377 +++++----- .../tpch-approved-plan/v1/spark33/19.txt | 128 ++-- .../tpch-approved-plan/v1/spark33/20.txt | 494 ++++++------ .../tpch-approved-plan/v1/spark33/21.txt | 467 ++++++------ .../tpch-approved-plan/v1/spark33/22.txt | 275 +++---- .../tpch-approved-plan/v1/spark33/3.txt | 230 +++--- .../tpch-approved-plan/v1/spark33/4.txt | 189 ++--- .../tpch-approved-plan/v1/spark33/5.txt | 533 +++++++------ .../tpch-approved-plan/v1/spark33/6.txt | 67 +- .../tpch-approved-plan/v1/spark33/7.txt | 506 +++++++------ .../tpch-approved-plan/v1/spark33/8.txt | 709 ++++++++++-------- .../tpch-approved-plan/v1/spark33/9.txt | 529 +++++++------ .../tpch-approved-plan/v1/spark34/1.txt | 83 +- .../tpch-approved-plan/v1/spark34/10.txt | 340 +++++---- .../tpch-approved-plan/v1/spark34/11.txt | 473 ++++++------ .../tpch-approved-plan/v1/spark34/12.txt | 185 ++--- .../tpch-approved-plan/v1/spark34/13.txt | 199 ++--- .../tpch-approved-plan/v1/spark34/14.txt | 132 ++-- .../tpch-approved-plan/v1/spark34/15.txt | 256 ++++--- .../tpch-approved-plan/v1/spark34/16.txt | 241 +++--- .../tpch-approved-plan/v1/spark34/17.txt | 216 +++--- .../tpch-approved-plan/v1/spark34/18.txt | 377 +++++----- .../tpch-approved-plan/v1/spark34/19.txt | 128 ++-- .../tpch-approved-plan/v1/spark34/20.txt | 494 ++++++------ .../tpch-approved-plan/v1/spark34/21.txt | 467 ++++++------ .../tpch-approved-plan/v1/spark34/22.txt | 275 +++---- .../tpch-approved-plan/v1/spark34/3.txt | 230 +++--- .../tpch-approved-plan/v1/spark34/4.txt | 189 ++--- .../tpch-approved-plan/v1/spark34/5.txt | 533 +++++++------ .../tpch-approved-plan/v1/spark34/6.txt | 67 +- .../tpch-approved-plan/v1/spark34/7.txt | 506 +++++++------ .../tpch-approved-plan/v1/spark34/8.txt | 709 ++++++++++-------- .../tpch-approved-plan/v1/spark34/9.txt | 529 +++++++------ 252 files changed, 33344 insertions(+), 29294 deletions(-) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt index 9316f6f8ebb5..69a930ea8545 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt index d67e373daf9b..8f93964d76f5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt @@ -1,64 +1,65 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ShuffleQueryStage (36) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- BroadcastQueryStage (8) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ NoopFilter (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- BroadcastQueryStage (18) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ NoopFilter (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- BroadcastQueryStage (27) - +- ColumnarBroadcastExchange (26) - +- ^ NoopFilter (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (44) + +- TakeOrderedAndProjectExecTransformer (43) + +- ^ ProjectExecTransformer (41) + +- ^ RegularHashAggregateExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ShuffleQueryStage (37) + +- ColumnarExchange (36) + +- VeloxAppendBatches (35) + +- ^ ProjectExecTransformer (33) + +- ^ RegularHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ NoopFilter (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ NoopFilter (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27) + +- ColumnarBroadcastExchange (26) + +- ^ NoopFilter (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- BroadcastHashJoin Inner BuildRight (61) - :- Project (57) - : +- BroadcastHashJoin Inner BuildRight (56) - : :- Project (51) - : : +- BroadcastHashJoin Inner BuildRight (50) - : : :- Filter (45) - : : : +- Scan parquet (44) - : : +- BroadcastExchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (55) - : +- Project (54) - : +- Filter (53) - : +- Scan parquet (52) - +- BroadcastExchange (60) - +- Filter (59) - +- Scan parquet (58) + TakeOrderedAndProject (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- BroadcastHashJoin Inner BuildRight (62) + :- Project (58) + : +- BroadcastHashJoin Inner BuildRight (57) + : :- Project (52) + : : +- BroadcastHashJoin Inner BuildRight (51) + : : :- Filter (46) + : : : +- Scan parquet (45) + : : +- BroadcastExchange (50) + : : +- Project (49) + : : +- Filter (48) + : : +- Scan parquet (47) + : +- BroadcastExchange (56) + : +- Project (55) + : +- Filter (54) + : +- Scan parquet (53) + +- BroadcastExchange (61) + +- Filter (60) + +- Scan parquet (59) (1) Scan parquet @@ -194,7 +195,7 @@ Join condition: None Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(32) FlushableHashAggregateExecTransformer +(32) RegularHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -209,155 +210,159 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(35) ColumnarExchange +(35) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(36) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(37) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(37) InputAdapter +(38) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(38) InputIteratorTransformer +(39) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(40) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(40) ProjectExecTransformer +(41) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(41) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(43) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(43) VeloxColumnarToRowExec +(44) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(44) Scan parquet +(45) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(46) Scan parquet +(47) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(48) Project +(49) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(51) Project +(52) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(52) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(53) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(54) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(55) BroadcastExchange +(56) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(57) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(58) Scan parquet +(59) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(59) Filter +(60) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(60) BroadcastExchange +(61) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(61) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) Project +(63) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(63) HashAggregate +(64) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(64) Exchange +(65) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(66) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(66) TakeOrderedAndProject +(67) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt index f65747c37e65..0e98c4c0edd7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt @@ -1,55 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7) - : +- ColumnarBroadcastExchange (6) - : +- ^ NoopFilter (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ NoopFilter (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7) + : +- ColumnarBroadcastExchange (6) + : +- ^ NoopFilter (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ NoopFilter (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (57) - +- Exchange (56) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- BroadcastHashJoin Inner BuildRight (50) - :- Project (45) - : +- BroadcastHashJoin Inner BuildRight (44) - : :- Filter (40) - : : +- Scan parquet (39) - : +- BroadcastExchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- BroadcastExchange (49) - +- Project (48) - +- Filter (47) - +- Scan parquet (46) + Sort (58) + +- Exchange (57) + +- Filter (56) + +- HashAggregate (55) + +- Exchange (54) + +- HashAggregate (53) + +- Project (52) + +- BroadcastHashJoin Inner BuildRight (51) + :- Project (46) + : +- BroadcastHashJoin Inner BuildRight (45) + : :- Filter (41) + : : +- Scan parquet (40) + : +- BroadcastExchange (44) + : +- Filter (43) + : +- Scan parquet (42) + +- BroadcastExchange (50) + +- Project (49) + +- Filter (48) + +- Scan parquet (47) (1) Scan parquet @@ -187,124 +188,128 @@ Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery su Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(34) InputAdapter +(35) InputAdapter Input [2]: [ps_partkey#X, value#X] -(35) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(36) SortExecTransformer +(37) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(38) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(39) Scan parquet +(40) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(41) Scan parquet +(42) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(42) Filter +(43) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(43) BroadcastExchange +(44) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(45) Project +(46) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(46) Scan parquet +(47) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(48) Project +(49) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(51) Project +(52) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(52) HashAggregate +(53) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(53) Exchange +(54) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(55) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(55) Filter +(56) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(56) Exchange +(57) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(58) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(58) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt index f1b7bbfe6acb..d4942783d0ff 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt @@ -1,41 +1,42 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (43) +- == Final Plan == - VeloxColumnarToRowExec (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (29) + +- ^ SortExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- VeloxAppendBatches (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) + Sort (42) + +- Exchange (41) + +- HashAggregate (40) + +- Exchange (39) + +- HashAggregate (38) + +- Project (37) + +- BroadcastHashJoin Inner BuildLeft (36) + :- BroadcastExchange (32) + : +- Filter (31) + : +- Scan parquet (30) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -131,96 +132,100 @@ Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_ Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) SortExecTransformer +(27) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(27) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(30) Filter +(31) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(31) BroadcastExchange +(32) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(32) Scan parquet +(33) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(33) Filter +(34) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(34) Project +(35) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(35) BroadcastHashJoin +(36) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) Project +(37) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(37) HashAggregate +(38) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(38) Exchange +(39) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(40) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(40) Exchange +(41) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(42) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(43) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt index 12147d4b3197..4320ccecc911 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt @@ -1,49 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ NoopFilter (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ NoopFilter (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- BroadcastHashJoin LeftOuter BuildRight (42) - :- Scan parquet (37) - +- BroadcastExchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- BroadcastHashJoin LeftOuter BuildRight (43) + :- Scan parquet (38) + +- BroadcastExchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -174,109 +175,113 @@ Results [2]: [c_count#X, count(1)#X AS custdist#X] Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(33) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(35) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(38) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(44) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) Exchange +(46) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(48) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(49) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(50) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(51) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(52) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt index 87dc45754b66..b79d92ba4257 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt @@ -1,35 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- BroadcastHashJoin Inner BuildRight (29) - :- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- BroadcastExchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- BroadcastHashJoin Inner BuildRight (30) + :- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- BroadcastExchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -85,7 +86,7 @@ Join condition: None Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] @@ -96,95 +97,99 @@ Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(23) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(25) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) Scan parquet +(27) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(28) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) BroadcastExchange +(29) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastHashJoin +(30) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(30) Project +(31) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(32) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) Exchange +(33) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] -(34) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt index 0bfcd38f02d2..50fc5e0f6630 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt @@ -1,43 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- Project (41) - +- BroadcastHashJoin Inner BuildLeft (40) - :- BroadcastExchange (32) - : +- Filter (31) - : +- Scan parquet (30) - +- Filter (39) - +- HashAggregate (38) - +- Exchange (37) - +- HashAggregate (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (44) + +- Exchange (43) + +- Project (42) + +- BroadcastHashJoin Inner BuildLeft (41) + :- BroadcastExchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Filter (40) + +- HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -137,100 +138,104 @@ Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_ Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) SortExecTransformer +(28) SortExecTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(30) Scan parquet +(31) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(32) BroadcastExchange +(33) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(33) Scan parquet +(34) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(34) Filter +(35) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(35) Project +(36) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(36) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(37) Exchange +(38) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) HashAggregate +(39) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(39) Filter +(40) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(40) BroadcastHashJoin +(41) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(41) Project +(42) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(42) Exchange +(43) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(44) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt index ccc5488215a6..ed3433681532 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt @@ -1,53 +1,54 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (57) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ShuffleQueryStage (30) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ NoopFilter (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ NoopFilter (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- BroadcastHashJoin Inner BuildRight (46) - :- BroadcastHashJoin LeftAnti BuildRight (42) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (45) - +- Filter (44) - +- Scan parquet (43) + Sort (56) + +- Exchange (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- BroadcastHashJoin Inner BuildRight (47) + :- BroadcastHashJoin LeftAnti BuildRight (43) + : :- Filter (38) + : : +- Scan parquet (37) + : +- BroadcastExchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- BroadcastExchange (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -175,134 +176,138 @@ Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_c Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) ColumnarExchange +(29) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(30) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(36) Scan parquet +(37) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(37) Filter +(38) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(38) Scan parquet +(39) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(39) Filter +(40) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(40) Project +(41) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(43) Scan parquet +(44) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(45) BroadcastExchange +(46) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(47) Project +(48) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(48) HashAggregate +(49) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(49) Exchange +(50) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(51) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(51) HashAggregate +(52) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(52) Exchange +(53) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(54) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(56) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(56) AdaptiveSparkPlan +(57) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/17.txt index 9d81e1d5053f..1b258da06f20 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/17.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (14) - +- ^ ProjectExecTransformer (12) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- RowToVeloxColumnar (2) - +- LocalTableScan (1) + VeloxColumnarToRowExec (15) + +- ^ ProjectExecTransformer (13) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ RegularHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == - HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin Inner BuildRight (30) - :- Project (22) - : +- BroadcastHashJoin Inner BuildRight (21) - : :- Filter (16) - : : +- Scan parquet (15) - : +- BroadcastExchange (20) - : +- Project (19) - : +- Filter (18) - : +- Scan parquet (17) - +- BroadcastExchange (29) - +- Filter (28) - +- HashAggregate (27) - +- Exchange (26) - +- HashAggregate (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin Inner BuildRight (31) + :- Project (23) + : +- BroadcastHashJoin Inner BuildRight (22) + : :- Filter (17) + : : +- Scan parquet (16) + : +- BroadcastExchange (21) + : +- Project (20) + : +- Filter (19) + : +- Scan parquet (18) + +- BroadcastExchange (30) + +- Filter (29) + +- HashAggregate (28) + +- Exchange (27) + +- HashAggregate (26) + +- Filter (25) + +- Scan parquet (24) (1) LocalTableScan @@ -47,7 +48,7 @@ Input [1]: [l_extendedprice#X] (4) InputIteratorTransformer Input [1]: [l_extendedprice#X] -(5) FlushableHashAggregateExecTransformer +(5) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] @@ -58,141 +59,145 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(8) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [2]: [sum#X, isEmpty#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(14) VeloxColumnarToRowExec +(15) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(15) Scan parquet +(16) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(16) Filter +(17) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(17) Scan parquet +(18) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(18) Filter +(19) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(19) Project +(20) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(20) BroadcastExchange +(21) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastHashJoin +(22) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(22) Project +(23) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(23) Scan parquet +(24) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(25) HashAggregate +(26) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(26) Exchange +(27) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) HashAggregate +(28) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(28) Filter +(29) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(31) Project +(32) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(32) HashAggregate +(33) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] -(35) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt index e582fd47cf55..505beebed868 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt @@ -1,80 +1,82 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- TakeOrderedAndProjectExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ShuffleQueryStage (47) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- BroadcastQueryStage (5) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ NoopFilter (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- BroadcastQueryStage (38) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- BroadcastQueryStage (32) - +- ReusedExchange (31) + VeloxColumnarToRowExec (55) + +- TakeOrderedAndProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49) + +- ColumnarExchange (48) + +- VeloxAppendBatches (47) + +- ^ ProjectExecTransformer (45) + +- ^ RegularHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) + :- ^ ProjectExecTransformer (29) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (28) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (27) + : :- ^ NoopFilter (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (26) + : +- BroadcastQueryStage (24) + : +- ColumnarBroadcastExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FilterExecTransformer (20) + : +- ^ RegularHashAggregateExecTransformer (19) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ RegularHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (41) + +- BroadcastQueryStage (39) + +- ColumnarBroadcastExchange (38) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (36) + :- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (35) + +- BroadcastQueryStage (33) + +- ReusedExchange (32) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (68) - : +- BroadcastHashJoin Inner BuildLeft (67) - : :- BroadcastExchange (56) - : : +- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastHashJoin LeftSemi BuildRight (66) - : :- Filter (58) - : : +- Scan parquet (57) - : +- BroadcastExchange (65) - : +- Project (64) - : +- Filter (63) - : +- HashAggregate (62) - : +- Exchange (61) - : +- HashAggregate (60) - : +- Scan parquet (59) - +- BroadcastExchange (79) - +- BroadcastHashJoin LeftSemi BuildRight (78) - :- Filter (70) - : +- Scan parquet (69) - +- BroadcastExchange (77) - +- Project (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Scan parquet (71) + TakeOrderedAndProject (87) + +- HashAggregate (86) + +- Exchange (85) + +- HashAggregate (84) + +- Project (83) + +- BroadcastHashJoin Inner BuildRight (82) + :- Project (70) + : +- BroadcastHashJoin Inner BuildLeft (69) + : :- BroadcastExchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- BroadcastHashJoin LeftSemi BuildRight (68) + : :- Filter (60) + : : +- Scan parquet (59) + : +- BroadcastExchange (67) + : +- Project (66) + : +- Filter (65) + : +- HashAggregate (64) + : +- Exchange (63) + : +- HashAggregate (62) + : +- Scan parquet (61) + +- BroadcastExchange (81) + +- BroadcastHashJoin LeftSemi BuildRight (80) + :- Filter (72) + : +- Scan parquet (71) + +- BroadcastExchange (79) + +- Project (78) + +- Filter (77) + +- HashAggregate (76) + +- Exchange (75) + +- HashAggregate (74) + +- Scan parquet (73) (1) Scan parquet @@ -123,7 +125,7 @@ Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) FlushableHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] @@ -138,333 +140,341 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [1]: [l_orderkey#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [1]: [l_orderkey#X] -(26) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(27) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(30) NoopFilter +(31) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(31) ReusedExchange [Reuses operator id: 22] +(32) ReusedExchange [Reuses operator id: 23] Output [1]: [l_orderkey#X] -(32) BroadcastQueryStage +(33) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [1]: [l_orderkey#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [l_orderkey#X] -(35) BroadcastHashJoinExecTransformer +(36) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(37) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(39) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(41) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(43) FlushableHashAggregateExecTransformer +(44) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(47) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: X + +(48) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(49) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(50) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(51) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(51) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(52) TakeOrderedAndProjectExecTransformer +(54) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(53) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(54) Scan parquet +(56) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(56) BroadcastExchange +(58) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(59) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(58) Filter +(60) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(59) Scan parquet +(61) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(60) HashAggregate +(62) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(61) Exchange +(63) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(63) Filter +(65) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(64) Project +(66) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(65) BroadcastExchange +(67) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(67) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(68) Project +(70) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(69) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(71) Scan parquet +(73) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(74) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(75) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(76) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(77) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(78) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) BroadcastExchange +(79) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(79) BroadcastExchange +(81) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(81) Project +(83) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(82) HashAggregate +(84) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(83) Exchange +(85) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(86) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(85) TakeOrderedAndProject +(87) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(86) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt index 72a9f74f86ce..f4c3b231f1ba 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt @@ -1,34 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (32) - +- Exchange (31) - +- HashAggregate (30) - +- Project (29) - +- BroadcastHashJoin Inner BuildRight (28) - :- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- BroadcastExchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -84,7 +85,7 @@ Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -95,91 +96,95 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(20) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(22) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(23) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(24) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(25) Scan parquet +(26) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(27) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) BroadcastExchange +(28) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(30) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(31) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) Exchange +(32) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(32) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(33) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt index e09911ea6ad8..426a01b76051 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt @@ -1,97 +1,99 @@ == Physical Plan == -AdaptiveSparkPlan (107) +AdaptiveSparkPlan (109) +- == Final Plan == - VeloxColumnarToRowExec (70) - +- ^ SortExecTransformer (68) - +- ^ InputIteratorTransformer (67) - +- ShuffleQueryStage (65) - +- ColumnarExchange (64) - +- ^ ProjectExecTransformer (62) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (61) - :- ^ ProjectExecTransformer (52) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (51) - : :- ^ InputIteratorTransformer (9) - : : +- AQEShuffleRead (7) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (50) - : +- BroadcastQueryStage (48) - : +- ColumnarBroadcastExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) - : :- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) - : : :- ^ NoopFilter (11) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (19) - : : +- BroadcastQueryStage (17) - : : +- ColumnarBroadcastExchange (16) - : : +- ^ ProjectExecTransformer (14) - : : +- ^ NoopFilter (13) - : : +- ^ Scan parquet (12) - : +- ^ FilterExecTransformer (43) - : +- ^ ProjectExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ InputIteratorTransformer (40) - : +- ShuffleQueryStage (38) - : +- ColumnarExchange (37) - : +- ^ ProjectExecTransformer (35) - : +- ^ FlushableHashAggregateExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) - : :- ^ ProjectExecTransformer (28) - : : +- ^ NoopFilter (27) - : : +- ^ Scan parquet (26) - : +- ^ InputIteratorTransformer (32) - : +- BroadcastQueryStage (30) - : +- ReusedExchange (29) - +- ^ InputIteratorTransformer (60) - +- BroadcastQueryStage (58) - +- ColumnarBroadcastExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ NoopFilter (54) - +- ^ Scan parquet (53) + VeloxColumnarToRowExec (72) + +- ^ SortExecTransformer (70) + +- ^ InputIteratorTransformer (69) + +- ShuffleQueryStage (67) + +- ColumnarExchange (66) + +- VeloxAppendBatches (65) + +- ^ ProjectExecTransformer (63) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (62) + :- ^ ProjectExecTransformer (53) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (52) + : :- ^ InputIteratorTransformer (10) + : : +- AQEShuffleRead (8) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (51) + : +- BroadcastQueryStage (49) + : +- ColumnarBroadcastExchange (48) + : +- ^ ProjectExecTransformer (46) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (45) + : :- ^ InputIteratorTransformer (26) + : : +- BroadcastQueryStage (24) + : : +- ColumnarBroadcastExchange (23) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (21) + : : :- ^ NoopFilter (12) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (20) + : : +- BroadcastQueryStage (18) + : : +- ColumnarBroadcastExchange (17) + : : +- ^ ProjectExecTransformer (15) + : : +- ^ NoopFilter (14) + : : +- ^ Scan parquet (13) + : +- ^ FilterExecTransformer (44) + : +- ^ ProjectExecTransformer (43) + : +- ^ RegularHashAggregateExecTransformer (42) + : +- ^ InputIteratorTransformer (41) + : +- ShuffleQueryStage (39) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ FlushableHashAggregateExecTransformer (35) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (34) + : :- ^ ProjectExecTransformer (29) + : : +- ^ NoopFilter (28) + : : +- ^ Scan parquet (27) + : +- ^ InputIteratorTransformer (33) + : +- BroadcastQueryStage (31) + : +- ReusedExchange (30) + +- ^ InputIteratorTransformer (61) + +- BroadcastQueryStage (59) + +- ColumnarBroadcastExchange (58) + +- ^ ProjectExecTransformer (56) + +- ^ NoopFilter (55) + +- ^ Scan parquet (54) +- == Initial Plan == - Sort (106) - +- Exchange (105) - +- Project (104) - +- BroadcastHashJoin Inner BuildRight (103) - :- Project (98) - : +- ShuffledHashJoin LeftSemi BuildRight (97) - : :- Exchange (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (96) - : +- Project (95) - : +- BroadcastHashJoin Inner BuildLeft (94) - : :- BroadcastExchange (81) - : : +- BroadcastHashJoin LeftSemi BuildRight (80) - : : :- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (79) - : : +- Project (78) - : : +- Filter (77) - : : +- Scan parquet (76) - : +- Filter (93) - : +- HashAggregate (92) - : +- Exchange (91) - : +- HashAggregate (90) - : +- BroadcastHashJoin LeftSemi BuildRight (89) - : :- Project (84) - : : +- Filter (83) - : : +- Scan parquet (82) - : +- BroadcastExchange (88) - : +- Project (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (102) - +- Project (101) - +- Filter (100) - +- Scan parquet (99) + Sort (108) + +- Exchange (107) + +- Project (106) + +- BroadcastHashJoin Inner BuildRight (105) + :- Project (100) + : +- ShuffledHashJoin LeftSemi BuildRight (99) + : :- Exchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- Exchange (98) + : +- Project (97) + : +- BroadcastHashJoin Inner BuildLeft (96) + : :- BroadcastExchange (83) + : : +- BroadcastHashJoin LeftSemi BuildRight (82) + : : :- Filter (77) + : : : +- Scan parquet (76) + : : +- BroadcastExchange (81) + : : +- Project (80) + : : +- Filter (79) + : : +- Scan parquet (78) + : +- Filter (95) + : +- HashAggregate (94) + : +- Exchange (93) + : +- HashAggregate (92) + : +- BroadcastHashJoin LeftSemi BuildRight (91) + : :- Project (86) + : : +- Filter (85) + : : +- Scan parquet (84) + : +- BroadcastExchange (90) + : +- Project (89) + : +- Filter (88) + : +- Scan parquet (87) + +- BroadcastExchange (104) + +- Project (103) + +- Filter (102) + +- Scan parquet (101) (1) Scan parquet @@ -113,448 +115,456 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) AQEShuffleRead +(8) AQEShuffleRead Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: local -(8) InputAdapter +(9) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(10) Scan parquet +(11) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(11) NoopFilter +(12) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) Scan parquet +(13) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(13) NoopFilter +(14) NoopFilter Input [2]: [p_partkey#X, p_name#X] Arguments: [p_partkey#X, p_name#X] -(14) ProjectExecTransformer +(15) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(15) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(16) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(17) BroadcastQueryStage +(18) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [1]: [p_partkey#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [1]: [p_partkey#X] -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(26) Scan parquet +(27) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(27) NoopFilter +(28) NoopFilter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(29) ReusedExchange [Reuses operator id: 16] +(30) ReusedExchange [Reuses operator id: 17] Output [1]: [p_partkey#X] -(30) BroadcastQueryStage +(31) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [1]: [p_partkey#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [1]: [p_partkey#X] -(33) BroadcastHashJoinExecTransformer +(34) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(34) FlushableHashAggregateExecTransformer +(35) FlushableHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(36) ProjectExecTransformer Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(38) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(39) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(42) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) FilterExecTransformer +(44) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(44) BroadcastHashJoinExecTransformer +(45) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(45) ProjectExecTransformer +(46) ProjectExecTransformer Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(46) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] Arguments: false -(47) ColumnarBroadcastExchange +(48) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(48) BroadcastQueryStage +(49) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(49) InputAdapter +(50) InputAdapter Input [1]: [ps_suppkey#X] -(50) InputIteratorTransformer +(51) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(51) BroadcastHashJoinExecTransformer +(52) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(52) ProjectExecTransformer +(53) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(53) Scan parquet +(54) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(54) NoopFilter +(55) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(55) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(56) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(57) ColumnarBroadcastExchange +(58) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(58) BroadcastQueryStage +(59) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(59) InputAdapter +(60) InputAdapter Input [1]: [n_nationkey#X] -(60) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [n_nationkey#X] -(61) BroadcastHashJoinExecTransformer +(62) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) ProjectExecTransformer +(63) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(63) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(64) ColumnarExchange +(65) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(66) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(65) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(66) InputAdapter +(68) InputAdapter Input [2]: [s_name#X, s_address#X] -(67) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [s_name#X, s_address#X] -(68) SortExecTransformer +(70) SortExecTransformer Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(69) WholeStageCodegenTransformer (X) +(71) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(70) VeloxColumnarToRowExec +(72) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(71) Scan parquet +(73) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(72) Filter +(74) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(73) Exchange +(75) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(76) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(75) Filter +(77) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(76) Scan parquet +(78) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(77) Filter +(79) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(78) Project +(80) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(79) BroadcastExchange +(81) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(80) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(81) BroadcastExchange +(83) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) Scan parquet +(84) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(83) Filter +(85) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(84) Project +(86) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(85) Scan parquet +(87) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(86) Filter +(88) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(87) Project +(89) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(88) BroadcastExchange +(90) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(90) HashAggregate +(92) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(91) Exchange +(93) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) HashAggregate +(94) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(93) Filter +(95) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(94) BroadcastHashJoin +(96) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(95) Project +(97) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(96) Exchange +(98) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(99) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(98) Project +(100) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(99) Scan parquet +(101) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(100) Filter +(102) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(101) Project +(103) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(102) BroadcastExchange +(104) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(103) BroadcastHashJoin +(105) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(104) Project +(106) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(105) Exchange +(107) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Sort +(108) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(107) AdaptiveSparkPlan +(109) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt index 10e08cfd60b5..10cf069932d7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt @@ -1,85 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (92) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (59) - +- TakeOrderedAndProjectExecTransformer (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- BroadcastQueryStage (5) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- BroadcastQueryStage (14) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ NoopFilter (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (60) + +- TakeOrderedAndProjectExecTransformer (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ NoopFilter (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (81) - : +- BroadcastHashJoin Inner BuildRight (80) - : :- Project (75) - : : +- BroadcastHashJoin Inner BuildLeft (74) - : : :- BroadcastExchange (62) - : : : +- Filter (61) - : : : +- Scan parquet (60) - : : +- BroadcastHashJoin LeftAnti BuildRight (73) - : : :- BroadcastHashJoin LeftSemi BuildRight (68) - : : : :- Project (65) - : : : : +- Filter (64) - : : : : +- Scan parquet (63) - : : : +- BroadcastExchange (67) - : : : +- Scan parquet (66) - : : +- BroadcastExchange (72) - : : +- Project (71) - : : +- Filter (70) - : : +- Scan parquet (69) - : +- BroadcastExchange (79) - : +- Project (78) - : +- Filter (77) - : +- Scan parquet (76) - +- BroadcastExchange (85) - +- Project (84) - +- Filter (83) - +- Scan parquet (82) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- BroadcastHashJoin Inner BuildRight (87) + :- Project (82) + : +- BroadcastHashJoin Inner BuildRight (81) + : :- Project (76) + : : +- BroadcastHashJoin Inner BuildLeft (75) + : : :- BroadcastExchange (63) + : : : +- Filter (62) + : : : +- Scan parquet (61) + : : +- BroadcastHashJoin LeftAnti BuildRight (74) + : : :- BroadcastHashJoin LeftSemi BuildRight (69) + : : : :- Project (66) + : : : : +- Filter (65) + : : : : +- Scan parquet (64) + : : : +- BroadcastExchange (68) + : : : +- Scan parquet (67) + : : +- BroadcastExchange (73) + : : +- Project (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- BroadcastExchange (80) + : +- Project (79) + : +- Filter (78) + : +- Scan parquet (77) + +- BroadcastExchange (86) + +- Project (85) + +- Filter (84) + +- Scan parquet (83) (1) Scan parquet @@ -286,7 +287,7 @@ Join condition: None Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(49) FlushableHashAggregateExecTransformer +(49) RegularHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] @@ -301,194 +302,198 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(53) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [2]: [s_name#X, count#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(58) TakeOrderedAndProjectExecTransformer +(59) TakeOrderedAndProjectExecTransformer Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X], 0 -(59) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(60) Scan parquet +(61) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(61) Filter +(62) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(62) BroadcastExchange +(63) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(63) Scan parquet +(64) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(64) Filter +(65) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(65) Project +(66) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(67) BroadcastExchange +(68) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(68) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(69) Scan parquet +(70) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(70) Filter +(71) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(71) Project +(72) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(72) BroadcastExchange +(73) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(74) BroadcastHashJoin +(75) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(75) Project +(76) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(76) Scan parquet +(77) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(77) Filter +(78) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(78) Project +(79) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(79) BroadcastExchange +(80) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(80) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(81) Project +(82) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(82) Scan parquet +(83) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(83) Filter +(84) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(84) Project +(85) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(85) BroadcastExchange +(86) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(87) Project +(88) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(88) HashAggregate +(89) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(89) Exchange +(90) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(91) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(91) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(92) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt index a8c66bef1716..287d6c30505a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (39) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ShuffleQueryStage (21) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (27) + +- ^ SortExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ShuffleQueryStage (22) + +- ColumnarExchange (21) + +- VeloxAppendBatches (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (37) - +- Exchange (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- BroadcastHashJoin LeftAnti BuildRight (31) - :- Filter (28) - : +- Scan parquet (27) - +- BroadcastExchange (30) - +- Scan parquet (29) + Sort (38) + +- Exchange (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- BroadcastHashJoin LeftAnti BuildRight (32) + :- Filter (29) + : +- Scan parquet (28) + +- BroadcastExchange (31) + +- Scan parquet (30) (1) Scan parquet @@ -118,87 +119,91 @@ Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacc Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) ColumnarExchange +(20) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(21) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(22) InputAdapter +(23) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) SortExecTransformer +(25) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(27) Scan parquet +(28) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(28) Filter +(29) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(29) Scan parquet +(30) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(30) BroadcastExchange +(31) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(31) BroadcastHashJoin +(32) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(32) Project +(33) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(33) HashAggregate +(34) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(34) Exchange +(35) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(36) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(36) Exchange +(37) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Sort +(38) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(38) AdaptiveSparkPlan +(39) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt index b9b7951d7652..d92e785299d4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- TakeOrderedAndProjectExecTransformer (33) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- BroadcastQueryStage (6) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- BroadcastQueryStage (18) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ NoopFilter (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (35) + +- TakeOrderedAndProjectExecTransformer (34) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ RegularHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ NoopFilter (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildLeft (41) - : :- BroadcastExchange (38) - : : +- Project (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + TakeOrderedAndProject (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- BroadcastHashJoin Inner BuildRight (48) + :- Project (43) + : +- BroadcastHashJoin Inner BuildLeft (42) + : :- BroadcastExchange (39) + : : +- Project (38) + : : +- Filter (37) + : : +- Scan parquet (36) + : +- Filter (41) + : +- Scan parquet (40) + +- BroadcastExchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -144,7 +145,7 @@ Join condition: None Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) FlushableHashAggregateExecTransformer +(23) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -159,131 +160,135 @@ Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: X + +(27) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(33) TakeOrderedAndProjectExecTransformer +(34) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(34) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(35) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(36) Filter +(37) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(37) Project +(38) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(38) BroadcastExchange +(39) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) Scan parquet +(40) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(42) Project +(43) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(43) Scan parquet +(44) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(45) Project +(46) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) BroadcastExchange +(47) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(48) Project +(49) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(49) HashAggregate +(50) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(50) Exchange +(51) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(52) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(52) TakeOrderedAndProject +(53) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt index 97d18724dc9d..68bfa339986a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt @@ -1,43 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- BroadcastHashJoin LeftSemi BuildRight (37) - :- Project (32) - : +- Filter (31) - : +- Scan parquet (30) - +- BroadcastExchange (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (44) + +- Exchange (43) + +- HashAggregate (42) + +- Exchange (41) + +- HashAggregate (40) + +- Project (39) + +- BroadcastHashJoin LeftSemi BuildRight (38) + :- Project (33) + : +- Filter (32) + : +- Scan parquet (31) + +- BroadcastExchange (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -137,100 +138,104 @@ Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(27) SortExecTransformer +(28) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(30) Scan parquet +(31) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(31) Filter +(32) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(32) Project +(33) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(33) Scan parquet +(34) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(34) Filter +(35) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(35) Project +(36) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(36) BroadcastExchange +(37) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(37) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(38) Project +(39) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(39) HashAggregate +(40) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(40) Exchange +(41) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) HashAggregate +(42) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(42) Exchange +(43) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(44) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt index a9b993d0a078..612d61aac3b4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt @@ -1,91 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (100) +AdaptiveSparkPlan (101) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (99) - +- Exchange (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Project (71) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (92) - +- Project (91) - +- Filter (90) - +- Scan parquet (89) + Sort (100) + +- Exchange (99) + +- HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- BroadcastHashJoin Inner BuildRight (94) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Project (72) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (93) + +- Project (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -337,196 +338,200 @@ Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedpric Input [2]: [n_name#X, revenue#X] Arguments: false -(59) ColumnarExchange +(59) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(60) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(61) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(61) InputAdapter +(62) InputAdapter Input [2]: [n_name#X, revenue#X] -(62) InputIteratorTransformer +(63) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(63) SortExecTransformer +(64) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(65) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(67) Filter +(68) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(68) BroadcastExchange +(69) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(71) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(73) Project +(74) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(75) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Project +(79) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(79) Scan parquet +(80) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(81) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(81) BroadcastExchange +(82) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(83) Project +(84) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(84) Scan parquet +(85) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(86) BroadcastExchange +(87) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) Project +(89) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(89) Scan parquet +(90) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(91) Project +(92) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(92) BroadcastExchange +(93) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(93) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(94) Project +(95) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(95) HashAggregate +(96) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(96) Exchange +(97) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(98) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(98) Exchange +(99) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Sort +(100) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(100) AdaptiveSparkPlan +(101) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt index 7aae3dccfd9b..a5022343944f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt index b0d68672ca3a..3ed89ddff162 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt @@ -1,85 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ SortExecTransformer (58) - +- ^ InputIteratorTransformer (57) - +- ShuffleQueryStage (55) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ NoopFilter (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ NoopFilter (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34) - : +- ColumnarBroadcastExchange (33) - : +- ^ NoopFilter (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40) - +- ReusedExchange (39) + VeloxColumnarToRowExec (61) + +- ^ SortExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56) + +- ColumnarExchange (55) + +- VeloxAppendBatches (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ NoopFilter (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ NoopFilter (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40) + +- ReusedExchange (39) +- == Initial Plan == - Sort (92) - +- Exchange (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (63) - : : : : : +- Filter (62) - : : : : : +- Scan parquet (61) - : : : : +- Filter (65) - : : : : +- Scan parquet (64) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (85) - +- Filter (84) - +- Scan parquet (83) + Sort (93) + +- Exchange (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- BroadcastHashJoin Inner BuildRight (87) + :- Project (83) + : +- BroadcastHashJoin Inner BuildRight (82) + : :- Project (78) + : : +- BroadcastHashJoin Inner BuildRight (77) + : : :- Project (73) + : : : +- BroadcastHashJoin Inner BuildRight (72) + : : : :- Project (68) + : : : : +- BroadcastHashJoin Inner BuildLeft (67) + : : : : :- BroadcastExchange (64) + : : : : : +- Filter (63) + : : : : : +- Scan parquet (62) + : : : : +- Filter (66) + : : : : +- Scan parquet (65) + : : : +- BroadcastExchange (71) + : : : +- Filter (70) + : : : +- Scan parquet (69) + : : +- BroadcastExchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- BroadcastExchange (81) + : +- Filter (80) + : +- Scan parquet (79) + +- BroadcastExchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -307,188 +308,192 @@ Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) ColumnarExchange +(54) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(55) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(56) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(56) InputAdapter +(57) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(57) InputIteratorTransformer +(58) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) SortExecTransformer +(59) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(59) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(60) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(61) Scan parquet +(62) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(63) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(63) BroadcastExchange +(64) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(64) Scan parquet +(65) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(65) Filter +(66) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(66) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(67) Project +(68) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(68) Scan parquet +(69) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(69) Filter +(70) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(70) BroadcastExchange +(71) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(72) Project +(73) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(77) Project +(78) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(79) Filter +(80) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(80) BroadcastExchange +(81) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(83) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(87) Project +(88) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(88) HashAggregate +(89) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(89) Exchange +(90) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(91) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(91) Exchange +(92) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(93) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(93) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt index 7e20b94edaa7..f309fbdce259 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt @@ -1,116 +1,117 @@ == Physical Plan == -AdaptiveSparkPlan (129) +AdaptiveSparkPlan (130) +- == Final Plan == - VeloxColumnarToRowExec (84) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ NoopFilter (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ NoopFilter (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ NoopFilter (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ NoopFilter (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53) - : +- ColumnarBroadcastExchange (52) - : +- ^ NoopFilter (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ NoopFilter (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (85) + +- ^ SortExecTransformer (83) + +- ^ InputIteratorTransformer (82) + +- ShuffleQueryStage (80) + +- ColumnarExchange (79) + +- VeloxAppendBatches (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ShuffleQueryStage (72) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ NoopFilter (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ NoopFilter (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ NoopFilter (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ NoopFilter (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53) + : +- ColumnarBroadcastExchange (52) + : +- ^ NoopFilter (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ NoopFilter (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (128) - +- Exchange (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- BroadcastHashJoin Inner BuildRight (122) - :- Project (117) - : +- BroadcastHashJoin Inner BuildRight (116) - : :- Project (112) - : : +- BroadcastHashJoin Inner BuildRight (111) - : : :- Project (107) - : : : +- BroadcastHashJoin Inner BuildRight (106) - : : : :- Project (102) - : : : : +- BroadcastHashJoin Inner BuildRight (101) - : : : : :- Project (97) - : : : : : +- BroadcastHashJoin Inner BuildRight (96) - : : : : : :- Project (92) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) - : : : : : : :- BroadcastExchange (88) - : : : : : : : +- Project (87) - : : : : : : : +- Filter (86) - : : : : : : : +- Scan parquet (85) - : : : : : : +- Filter (90) - : : : : : : +- Scan parquet (89) - : : : : : +- BroadcastExchange (95) - : : : : : +- Filter (94) - : : : : : +- Scan parquet (93) - : : : : +- BroadcastExchange (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- BroadcastExchange (105) - : : : +- Filter (104) - : : : +- Scan parquet (103) - : : +- BroadcastExchange (110) - : : +- Filter (109) - : : +- Scan parquet (108) - : +- BroadcastExchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- BroadcastExchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (129) + +- Exchange (128) + +- HashAggregate (127) + +- Exchange (126) + +- HashAggregate (125) + +- Project (124) + +- BroadcastHashJoin Inner BuildRight (123) + :- Project (118) + : +- BroadcastHashJoin Inner BuildRight (117) + : :- Project (113) + : : +- BroadcastHashJoin Inner BuildRight (112) + : : :- Project (108) + : : : +- BroadcastHashJoin Inner BuildRight (107) + : : : :- Project (103) + : : : : +- BroadcastHashJoin Inner BuildRight (102) + : : : : :- Project (98) + : : : : : +- BroadcastHashJoin Inner BuildRight (97) + : : : : : :- Project (93) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (92) + : : : : : : :- BroadcastExchange (89) + : : : : : : : +- Project (88) + : : : : : : : +- Filter (87) + : : : : : : : +- Scan parquet (86) + : : : : : : +- Filter (91) + : : : : : : +- Scan parquet (90) + : : : : : +- BroadcastExchange (96) + : : : : : +- Filter (95) + : : : : : +- Scan parquet (94) + : : : : +- BroadcastExchange (101) + : : : : +- Filter (100) + : : : : +- Scan parquet (99) + : : : +- BroadcastExchange (106) + : : : +- Filter (105) + : : : +- Scan parquet (104) + : : +- BroadcastExchange (111) + : : +- Filter (110) + : : +- Scan parquet (109) + : +- BroadcastExchange (116) + : +- Filter (115) + : +- Scan parquet (114) + +- BroadcastExchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) (1) Scan parquet @@ -442,244 +443,248 @@ Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.000 Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) ColumnarExchange +(78) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(79) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(80) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(80) InputAdapter +(81) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(81) InputIteratorTransformer +(82) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(82) SortExecTransformer +(83) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(83) WholeStageCodegenTransformer (X) +(84) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(84) VeloxColumnarToRowExec +(85) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(85) Scan parquet +(86) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(87) Project +(88) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(88) BroadcastExchange +(89) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) Scan parquet +(90) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(92) Project +(93) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(93) Scan parquet +(94) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(94) Filter +(95) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(95) BroadcastExchange +(96) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(96) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(97) Project +(98) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(98) Scan parquet +(99) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(99) Filter +(100) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(100) BroadcastExchange +(101) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(101) BroadcastHashJoin +(102) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(102) Project +(103) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(103) Scan parquet +(104) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(104) Filter +(105) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(105) BroadcastExchange +(106) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(106) BroadcastHashJoin +(107) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(107) Project +(108) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(108) Scan parquet +(109) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(109) Filter +(110) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(110) BroadcastExchange +(111) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(111) BroadcastHashJoin +(112) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(112) Project +(113) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(113) Scan parquet +(114) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(114) Filter +(115) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(115) BroadcastExchange +(116) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(116) BroadcastHashJoin +(117) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(117) Project +(118) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(118) Scan parquet +(119) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(119) Filter +(120) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(120) Project +(121) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(121) BroadcastExchange +(122) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(122) BroadcastHashJoin +(123) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(123) Project +(124) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(124) HashAggregate +(125) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) Exchange +(126) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(127) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(127) Exchange +(128) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(129) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(129) AdaptiveSparkPlan +(130) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt index 8a30fbdfb152..0420b98c2c1f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt @@ -1,89 +1,90 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (99) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- ^ SortExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44) - +- ColumnarBroadcastExchange (43) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (65) + +- ^ SortExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ShuffleQueryStage (53) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (97) - +- Exchange (96) - +- HashAggregate (95) - +- Exchange (94) - +- HashAggregate (93) - +- Project (92) - +- BroadcastHashJoin Inner BuildRight (91) - :- Project (87) - : +- BroadcastHashJoin Inner BuildRight (86) - : :- Project (82) - : : +- BroadcastHashJoin Inner BuildRight (81) - : : :- Project (77) - : : : +- BroadcastHashJoin Inner BuildRight (76) - : : : :- Project (72) - : : : : +- BroadcastHashJoin Inner BuildLeft (71) - : : : : :- BroadcastExchange (68) - : : : : : +- Project (67) - : : : : : +- Filter (66) - : : : : : +- Scan parquet (65) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (75) - : : : +- Filter (74) - : : : +- Scan parquet (73) - : : +- BroadcastExchange (80) - : : +- Filter (79) - : : +- Scan parquet (78) - : +- BroadcastExchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- BroadcastExchange (90) - +- Filter (89) - +- Scan parquet (88) + Sort (98) + +- Exchange (97) + +- HashAggregate (96) + +- Exchange (95) + +- HashAggregate (94) + +- Project (93) + +- BroadcastHashJoin Inner BuildRight (92) + :- Project (88) + : +- BroadcastHashJoin Inner BuildRight (87) + : :- Project (83) + : : +- BroadcastHashJoin Inner BuildRight (82) + : : :- Project (78) + : : : +- BroadcastHashJoin Inner BuildRight (77) + : : : :- Project (73) + : : : : +- BroadcastHashJoin Inner BuildLeft (72) + : : : : :- BroadcastExchange (69) + : : : : : +- Project (68) + : : : : : +- Filter (67) + : : : : : +- Scan parquet (66) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (76) + : : : +- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- BroadcastExchange (86) + : +- Filter (85) + : +- Scan parquet (84) + +- BroadcastExchange (91) + +- Filter (90) + +- Scan parquet (89) (1) Scan parquet @@ -331,192 +332,196 @@ Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) ColumnarExchange +(58) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(59) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(60) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(60) InputAdapter +(61) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(61) InputIteratorTransformer +(62) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) SortExecTransformer +(63) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(63) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(64) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(67) Project +(68) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(69) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(72) Project +(73) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(77) Project +(78) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(79) Filter +(80) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(80) BroadcastExchange +(81) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(82) Project +(83) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(87) Project +(88) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(88) Scan parquet +(89) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(90) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(90) BroadcastExchange +(91) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(92) Project +(93) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(93) HashAggregate +(94) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(94) Exchange +(95) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) HashAggregate +(96) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(96) Exchange +(97) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(98) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(98) AdaptiveSparkPlan +(99) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt index 63b30bb5d26b..5ad7bc9d3675 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt index 18c2ef9bc473..4e657eb7e540 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt @@ -1,64 +1,65 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- BroadcastQueryStage (8), Statistics(X) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ NoopFilter (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- BroadcastQueryStage (18), Statistics(X) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ NoopFilter (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- BroadcastQueryStage (27), Statistics(X) - +- ColumnarBroadcastExchange (26) - +- ^ NoopFilter (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (44) + +- TakeOrderedAndProjectExecTransformer (43) + +- ^ ProjectExecTransformer (41) + +- ^ RegularHashAggregateExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- VeloxAppendBatches (35) + +- ^ ProjectExecTransformer (33) + +- ^ RegularHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ NoopFilter (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ NoopFilter (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ NoopFilter (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- BroadcastHashJoin Inner BuildRight (61) - :- Project (57) - : +- BroadcastHashJoin Inner BuildRight (56) - : :- Project (51) - : : +- BroadcastHashJoin Inner BuildRight (50) - : : :- Filter (45) - : : : +- Scan parquet (44) - : : +- BroadcastExchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (55) - : +- Project (54) - : +- Filter (53) - : +- Scan parquet (52) - +- BroadcastExchange (60) - +- Filter (59) - +- Scan parquet (58) + TakeOrderedAndProject (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- BroadcastHashJoin Inner BuildRight (62) + :- Project (58) + : +- BroadcastHashJoin Inner BuildRight (57) + : :- Project (52) + : : +- BroadcastHashJoin Inner BuildRight (51) + : : :- Filter (46) + : : : +- Scan parquet (45) + : : +- BroadcastExchange (50) + : : +- Project (49) + : : +- Filter (48) + : : +- Scan parquet (47) + : +- BroadcastExchange (56) + : +- Project (55) + : +- Filter (54) + : +- Scan parquet (53) + +- BroadcastExchange (61) + +- Filter (60) + +- Scan parquet (59) (1) Scan parquet @@ -194,7 +195,7 @@ Join condition: None Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(32) FlushableHashAggregateExecTransformer +(32) RegularHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -209,155 +210,159 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(35) ColumnarExchange +(35) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(36) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(37) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(37) InputAdapter +(38) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(38) InputIteratorTransformer +(39) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(40) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(40) ProjectExecTransformer +(41) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(41) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(43) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(43) VeloxColumnarToRowExec +(44) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(44) Scan parquet +(45) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(46) Scan parquet +(47) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(48) Project +(49) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(51) Project +(52) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(52) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(53) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(54) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(55) BroadcastExchange +(56) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(57) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(58) Scan parquet +(59) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(59) Filter +(60) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(60) BroadcastExchange +(61) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(61) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) Project +(63) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(63) HashAggregate +(64) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(64) Exchange +(65) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(66) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(66) TakeOrderedAndProject +(67) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt index 6afbee587ed2..5be334bc198e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt @@ -1,55 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7), Statistics(X) - : +- ColumnarBroadcastExchange (6) - : +- ^ NoopFilter (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17), Statistics(X) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ NoopFilter (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ NoopFilter (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ NoopFilter (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (57) - +- Exchange (56) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- BroadcastHashJoin Inner BuildRight (50) - :- Project (45) - : +- BroadcastHashJoin Inner BuildRight (44) - : :- Filter (40) - : : +- Scan parquet (39) - : +- BroadcastExchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- BroadcastExchange (49) - +- Project (48) - +- Filter (47) - +- Scan parquet (46) + Sort (58) + +- Exchange (57) + +- Filter (56) + +- HashAggregate (55) + +- Exchange (54) + +- HashAggregate (53) + +- Project (52) + +- BroadcastHashJoin Inner BuildRight (51) + :- Project (46) + : +- BroadcastHashJoin Inner BuildRight (45) + : :- Filter (41) + : : +- Scan parquet (40) + : +- BroadcastExchange (44) + : +- Filter (43) + : +- Scan parquet (42) + +- BroadcastExchange (50) + +- Project (49) + +- Filter (48) + +- Scan parquet (47) (1) Scan parquet @@ -187,350 +188,354 @@ Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery su Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(34) InputAdapter +(35) InputAdapter Input [2]: [ps_partkey#X, value#X] -(35) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(36) SortExecTransformer +(37) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(38) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(39) Scan parquet +(40) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(41) Scan parquet +(42) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(42) Filter +(43) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(43) BroadcastExchange +(44) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(45) Project +(46) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(46) Scan parquet +(47) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(48) Project +(49) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(51) Project +(52) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(52) HashAggregate +(53) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(53) Exchange +(54) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(55) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(55) Filter +(56) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(56) Exchange +(57) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(58) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(58) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 30 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (99) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (82) - +- ^ ProjectExecTransformer (80) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) - :- ^ ProjectExecTransformer (66) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) - : :- ^ NoopFilter (60) - : : +- ^ Scan parquet (59) - : +- ^ InputIteratorTransformer (64) - : +- BroadcastQueryStage (62), Statistics(X) - : +- ReusedExchange (61) - +- ^ InputIteratorTransformer (70) - +- BroadcastQueryStage (68), Statistics(X) - +- ReusedExchange (67) + VeloxColumnarToRowExec (83) + +- ^ ProjectExecTransformer (81) + +- ^ RegularHashAggregateExecTransformer (80) + +- ^ InputIteratorTransformer (79) + +- ShuffleQueryStage (77), Statistics(X) + +- ColumnarExchange (76) + +- ^ FlushableHashAggregateExecTransformer (74) + +- ^ ProjectExecTransformer (73) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (72) + :- ^ ProjectExecTransformer (67) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + : :- ^ NoopFilter (61) + : : +- ^ Scan parquet (60) + : +- ^ InputIteratorTransformer (65) + : +- BroadcastQueryStage (63), Statistics(X) + : +- ReusedExchange (62) + +- ^ InputIteratorTransformer (71) + +- BroadcastQueryStage (69), Statistics(X) + +- ReusedExchange (68) +- == Initial Plan == - HashAggregate (98) - +- Exchange (97) - +- HashAggregate (96) - +- Project (95) - +- BroadcastHashJoin Inner BuildRight (94) - :- Project (89) - : +- BroadcastHashJoin Inner BuildRight (88) - : :- Filter (84) - : : +- Scan parquet (83) - : +- BroadcastExchange (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (93) - +- Project (92) - +- Filter (91) - +- Scan parquet (90) - - -(59) Scan parquet + HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Project (96) + +- BroadcastHashJoin Inner BuildRight (95) + :- Project (90) + : +- BroadcastHashJoin Inner BuildRight (89) + : :- Filter (85) + : : +- Scan parquet (84) + : +- BroadcastExchange (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (94) + +- Project (93) + +- Filter (92) + +- Scan parquet (91) + + +(60) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(60) NoopFilter +(61) NoopFilter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(61) ReusedExchange [Reuses operator id: 6] +(62) ReusedExchange [Reuses operator id: 6] Output [2]: [s_suppkey#X, s_nationkey#X] -(62) BroadcastQueryStage +(63) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(63) InputAdapter +(64) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(64) InputIteratorTransformer +(65) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(65) BroadcastHashJoinExecTransformer +(66) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(66) ProjectExecTransformer +(67) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(67) ReusedExchange [Reuses operator id: 16] +(68) ReusedExchange [Reuses operator id: 16] Output [1]: [n_nationkey#X] -(68) BroadcastQueryStage +(69) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(69) InputAdapter +(70) InputAdapter Input [1]: [n_nationkey#X] -(70) InputIteratorTransformer +(71) InputIteratorTransformer Input [1]: [n_nationkey#X] -(71) BroadcastHashJoinExecTransformer +(72) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(72) ProjectExecTransformer +(73) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(73) FlushableHashAggregateExecTransformer +(74) FlushableHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(74) WholeStageCodegenTransformer (X) +(75) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(75) ColumnarExchange +(76) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(77) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(77) InputAdapter +(78) InputAdapter Input [2]: [sum#X, isEmpty#X] -(78) InputIteratorTransformer +(79) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(79) RegularHashAggregateExecTransformer +(80) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(80) ProjectExecTransformer +(81) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(81) WholeStageCodegenTransformer (X) +(82) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(82) VeloxColumnarToRowExec +(83) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(83) Scan parquet +(84) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(84) Filter +(85) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(85) Scan parquet +(86) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(87) BroadcastExchange +(88) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(89) Project +(90) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(90) Scan parquet +(91) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(91) Filter +(92) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(92) Project +(93) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(93) BroadcastExchange +(94) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(94) BroadcastHashJoin +(95) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(95) Project +(96) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(96) HashAggregate +(97) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(97) Exchange +(98) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(98) HashAggregate +(99) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(99) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt index c65916d66898..7104a3e8bc38 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt @@ -1,41 +1,42 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (43) +- == Final Plan == - VeloxColumnarToRowExec (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (29) + +- ^ SortExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- VeloxAppendBatches (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) + Sort (42) + +- Exchange (41) + +- HashAggregate (40) + +- Exchange (39) + +- HashAggregate (38) + +- Project (37) + +- BroadcastHashJoin Inner BuildLeft (36) + :- BroadcastExchange (32) + : +- Filter (31) + : +- Scan parquet (30) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -131,96 +132,100 @@ Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_ Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) SortExecTransformer +(27) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(27) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(30) Filter +(31) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(31) BroadcastExchange +(32) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(32) Scan parquet +(33) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(33) Filter +(34) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(34) Project +(35) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(35) BroadcastHashJoin +(36) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) Project +(37) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(37) HashAggregate +(38) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(38) Exchange +(39) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(40) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(40) Exchange +(41) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(42) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(43) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt index 9ff054f9314e..9658b4be0cc7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt @@ -1,49 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ NoopFilter (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ NoopFilter (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- BroadcastHashJoin LeftOuter BuildRight (42) - :- Scan parquet (37) - +- BroadcastExchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- BroadcastHashJoin LeftOuter BuildRight (43) + :- Scan parquet (38) + +- BroadcastExchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -174,109 +175,113 @@ Results [2]: [c_count#X, count(1)#X AS custdist#X] Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(33) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(35) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(38) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(44) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) Exchange +(46) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(48) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(49) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(50) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(51) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(52) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt index cc14a0347f1d..fbcaf99758c4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt @@ -1,35 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- BroadcastHashJoin Inner BuildRight (29) - :- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- BroadcastExchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- BroadcastHashJoin Inner BuildRight (30) + :- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- BroadcastExchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -85,7 +86,7 @@ Join condition: None Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] @@ -96,95 +97,99 @@ Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(23) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(25) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) Scan parquet +(27) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(28) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) BroadcastExchange +(29) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastHashJoin +(30) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(30) Project +(31) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(32) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) Exchange +(33) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] -(34) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt index 2d463a561ab6..91b117a86c00 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt @@ -1,42 +1,43 @@ == Physical Plan == -AdaptiveSparkPlan (41) +AdaptiveSparkPlan (42) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- AQEShuffleRead (25) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (27) + +- AQEShuffleRead (26) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (40) - +- Exchange (39) - +- Project (38) - +- BroadcastHashJoin Inner BuildLeft (37) - :- BroadcastExchange (29) - : +- Filter (28) - : +- Scan parquet (27) - +- Filter (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- Filter (31) - +- Scan parquet (30) + Sort (41) + +- Exchange (40) + +- Project (39) + +- BroadcastHashJoin Inner BuildLeft (38) + :- BroadcastExchange (30) + : +- Filter (29) + : +- Scan parquet (28) + +- Filter (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- Filter (32) + +- Scan parquet (31) (1) Scan parquet @@ -136,246 +137,250 @@ Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_ Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(25) AQEShuffleRead +(26) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(26) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) Scan parquet +(28) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(28) Filter +(29) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) Scan parquet +(31) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(32) Project +(33) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(33) HashAggregate +(34) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(34) Exchange +(35) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(36) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(36) Filter +(37) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(37) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(38) Project +(39) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(39) Exchange +(40) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) Sort +(41) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(41) AdaptiveSparkPlan +(42) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 19 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (66) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (57) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ ProjectExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ NoopFilter (43) - +- ^ Scan parquet (42) + VeloxColumnarToRowExec (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- ^ ProjectExecTransformer (47) + +- ^ FlushableHashAggregateExecTransformer (46) + +- ^ ProjectExecTransformer (45) + +- ^ NoopFilter (44) + +- ^ Scan parquet (43) +- == Initial Plan == - HashAggregate (65) - +- HashAggregate (64) - +- HashAggregate (63) - +- Exchange (62) - +- HashAggregate (61) - +- Project (60) - +- Filter (59) - +- Scan parquet (58) + HashAggregate (66) + +- HashAggregate (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- Project (61) + +- Filter (60) + +- Scan parquet (59) -(42) Scan parquet +(43) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(43) NoopFilter +(44) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(45) FlushableHashAggregateExecTransformer +(46) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(46) ProjectExecTransformer +(47) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(49) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(53) ProjectExecTransformer +(54) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(54) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(55) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(56) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(57) VeloxColumnarToRowExec +(58) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(58) Scan parquet +(59) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(59) Filter +(60) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(60) Project +(61) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(61) HashAggregate +(62) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(62) Exchange +(63) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(64) HashAggregate +(65) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(65) HashAggregate +(66) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(66) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt index c4d21c42e70b..da533a30dde7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt @@ -1,53 +1,54 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (57) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ NoopFilter (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ NoopFilter (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- BroadcastHashJoin Inner BuildRight (46) - :- BroadcastHashJoin LeftAnti BuildRight (42) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (45) - +- Filter (44) - +- Scan parquet (43) + Sort (56) + +- Exchange (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- BroadcastHashJoin Inner BuildRight (47) + :- BroadcastHashJoin LeftAnti BuildRight (43) + : :- Filter (38) + : : +- Scan parquet (37) + : +- BroadcastExchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- BroadcastExchange (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -175,134 +176,138 @@ Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_c Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) ColumnarExchange +(29) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(30) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(36) Scan parquet +(37) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(37) Filter +(38) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(38) Scan parquet +(39) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(39) Filter +(40) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(40) Project +(41) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(43) Scan parquet +(44) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(45) BroadcastExchange +(46) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(47) Project +(48) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(48) HashAggregate +(49) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(49) Exchange +(50) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(51) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(51) HashAggregate +(52) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(52) Exchange +(53) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(54) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(56) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(56) AdaptiveSparkPlan +(57) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/17.txt index 0b41400d5864..a57c908f0178 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/17.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (14) - +- ^ ProjectExecTransformer (12) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- RowToVeloxColumnar (2) - +- LocalTableScan (1) + VeloxColumnarToRowExec (15) + +- ^ ProjectExecTransformer (13) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ RegularHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == - HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin Inner BuildRight (30) - :- Project (22) - : +- BroadcastHashJoin Inner BuildRight (21) - : :- Filter (16) - : : +- Scan parquet (15) - : +- BroadcastExchange (20) - : +- Project (19) - : +- Filter (18) - : +- Scan parquet (17) - +- BroadcastExchange (29) - +- Filter (28) - +- HashAggregate (27) - +- Exchange (26) - +- HashAggregate (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin Inner BuildRight (31) + :- Project (23) + : +- BroadcastHashJoin Inner BuildRight (22) + : :- Filter (17) + : : +- Scan parquet (16) + : +- BroadcastExchange (21) + : +- Project (20) + : +- Filter (19) + : +- Scan parquet (18) + +- BroadcastExchange (30) + +- Filter (29) + +- HashAggregate (28) + +- Exchange (27) + +- HashAggregate (26) + +- Filter (25) + +- Scan parquet (24) (1) LocalTableScan @@ -47,7 +48,7 @@ Input [1]: [l_extendedprice#X] (4) InputIteratorTransformer Input [1]: [l_extendedprice#X] -(5) FlushableHashAggregateExecTransformer +(5) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] @@ -58,141 +59,145 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(8) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [2]: [sum#X, isEmpty#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(14) VeloxColumnarToRowExec +(15) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(15) Scan parquet +(16) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(16) Filter +(17) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(17) Scan parquet +(18) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(18) Filter +(19) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(19) Project +(20) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(20) BroadcastExchange +(21) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastHashJoin +(22) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(22) Project +(23) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(23) Scan parquet +(24) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(25) HashAggregate +(26) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(26) Exchange +(27) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) HashAggregate +(28) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(28) Filter +(29) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(31) Project +(32) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(32) HashAggregate +(33) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] -(35) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt index 8f03bd54cd2d..810b57c0f9bb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt @@ -1,80 +1,82 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- TakeOrderedAndProjectExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ NoopFilter (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23), Statistics(X) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- BroadcastQueryStage (32), Statistics(X) - +- ReusedExchange (31) + VeloxColumnarToRowExec (55) + +- TakeOrderedAndProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- VeloxAppendBatches (47) + +- ^ ProjectExecTransformer (45) + +- ^ RegularHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) + :- ^ ProjectExecTransformer (29) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (28) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (27) + : :- ^ NoopFilter (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (26) + : +- BroadcastQueryStage (24), Statistics(X) + : +- ColumnarBroadcastExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FilterExecTransformer (20) + : +- ^ RegularHashAggregateExecTransformer (19) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ RegularHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (41) + +- BroadcastQueryStage (39), Statistics(X) + +- ColumnarBroadcastExchange (38) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (36) + :- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (35) + +- BroadcastQueryStage (33), Statistics(X) + +- ReusedExchange (32) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (68) - : +- BroadcastHashJoin Inner BuildLeft (67) - : :- BroadcastExchange (56) - : : +- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastHashJoin LeftSemi BuildRight (66) - : :- Filter (58) - : : +- Scan parquet (57) - : +- BroadcastExchange (65) - : +- Project (64) - : +- Filter (63) - : +- HashAggregate (62) - : +- Exchange (61) - : +- HashAggregate (60) - : +- Scan parquet (59) - +- BroadcastExchange (79) - +- BroadcastHashJoin LeftSemi BuildRight (78) - :- Filter (70) - : +- Scan parquet (69) - +- BroadcastExchange (77) - +- Project (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Scan parquet (71) + TakeOrderedAndProject (87) + +- HashAggregate (86) + +- Exchange (85) + +- HashAggregate (84) + +- Project (83) + +- BroadcastHashJoin Inner BuildRight (82) + :- Project (70) + : +- BroadcastHashJoin Inner BuildLeft (69) + : :- BroadcastExchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- BroadcastHashJoin LeftSemi BuildRight (68) + : :- Filter (60) + : : +- Scan parquet (59) + : +- BroadcastExchange (67) + : +- Project (66) + : +- Filter (65) + : +- HashAggregate (64) + : +- Exchange (63) + : +- HashAggregate (62) + : +- Scan parquet (61) + +- BroadcastExchange (81) + +- BroadcastHashJoin LeftSemi BuildRight (80) + :- Filter (72) + : +- Scan parquet (71) + +- BroadcastExchange (79) + +- Project (78) + +- Filter (77) + +- HashAggregate (76) + +- Exchange (75) + +- HashAggregate (74) + +- Scan parquet (73) (1) Scan parquet @@ -123,7 +125,7 @@ Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) FlushableHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] @@ -138,333 +140,341 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [1]: [l_orderkey#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [1]: [l_orderkey#X] -(26) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(27) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(30) NoopFilter +(31) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(31) ReusedExchange [Reuses operator id: 22] +(32) ReusedExchange [Reuses operator id: 23] Output [1]: [l_orderkey#X] -(32) BroadcastQueryStage +(33) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [1]: [l_orderkey#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [l_orderkey#X] -(35) BroadcastHashJoinExecTransformer +(36) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(37) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(39) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(41) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(43) FlushableHashAggregateExecTransformer +(44) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(47) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: X + +(48) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(49) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(50) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(51) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(51) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(52) TakeOrderedAndProjectExecTransformer +(54) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(53) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(54) Scan parquet +(56) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(56) BroadcastExchange +(58) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(59) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(58) Filter +(60) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(59) Scan parquet +(61) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(60) HashAggregate +(62) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(61) Exchange +(63) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(63) Filter +(65) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(64) Project +(66) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(65) BroadcastExchange +(67) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(67) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(68) Project +(70) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(69) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(71) Scan parquet +(73) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(74) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(75) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(76) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(77) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(78) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) BroadcastExchange +(79) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(79) BroadcastExchange +(81) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(81) Project +(83) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(82) HashAggregate +(84) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(83) Exchange +(85) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(86) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(85) TakeOrderedAndProject +(87) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(86) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt index 1fcd294716ef..46bbc58abcde 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt @@ -1,34 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (32) - +- Exchange (31) - +- HashAggregate (30) - +- Project (29) - +- BroadcastHashJoin Inner BuildRight (28) - :- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- BroadcastExchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -84,7 +85,7 @@ Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -95,91 +96,95 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(20) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(22) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(23) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(24) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(25) Scan parquet +(26) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(27) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) BroadcastExchange +(28) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(30) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(31) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) Exchange +(32) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(32) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(33) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt index ee30d646767e..3ac12224cffa 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt @@ -1,96 +1,98 @@ == Physical Plan == -AdaptiveSparkPlan (104) +AdaptiveSparkPlan (106) +- == Final Plan == - VeloxColumnarToRowExec (67) - +- AQEShuffleRead (66) - +- ShuffleQueryStage (65), Statistics(X) - +- ColumnarExchange (64) - +- ^ ProjectExecTransformer (62) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (61) - :- ^ ProjectExecTransformer (52) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (51) - : :- ^ InputIteratorTransformer (9) - : : +- AQEShuffleRead (7) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (50) - : +- BroadcastQueryStage (48), Statistics(X) - : +- ColumnarBroadcastExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) - : :- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) - : : :- ^ NoopFilter (11) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (19) - : : +- BroadcastQueryStage (17), Statistics(X) - : : +- ColumnarBroadcastExchange (16) - : : +- ^ ProjectExecTransformer (14) - : : +- ^ NoopFilter (13) - : : +- ^ Scan parquet (12) - : +- ^ FilterExecTransformer (43) - : +- ^ ProjectExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ InputIteratorTransformer (40) - : +- ShuffleQueryStage (38), Statistics(X) - : +- ColumnarExchange (37) - : +- ^ ProjectExecTransformer (35) - : +- ^ FlushableHashAggregateExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) - : :- ^ ProjectExecTransformer (28) - : : +- ^ NoopFilter (27) - : : +- ^ Scan parquet (26) - : +- ^ InputIteratorTransformer (32) - : +- BroadcastQueryStage (30), Statistics(X) - : +- ReusedExchange (29) - +- ^ InputIteratorTransformer (60) - +- BroadcastQueryStage (58), Statistics(X) - +- ColumnarBroadcastExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ NoopFilter (54) - +- ^ Scan parquet (53) + VeloxColumnarToRowExec (69) + +- AQEShuffleRead (68) + +- ShuffleQueryStage (67), Statistics(X) + +- ColumnarExchange (66) + +- VeloxAppendBatches (65) + +- ^ ProjectExecTransformer (63) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (62) + :- ^ ProjectExecTransformer (53) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (52) + : :- ^ InputIteratorTransformer (10) + : : +- AQEShuffleRead (8) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (51) + : +- BroadcastQueryStage (49), Statistics(X) + : +- ColumnarBroadcastExchange (48) + : +- ^ ProjectExecTransformer (46) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (45) + : :- ^ InputIteratorTransformer (26) + : : +- BroadcastQueryStage (24), Statistics(X) + : : +- ColumnarBroadcastExchange (23) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (21) + : : :- ^ NoopFilter (12) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (20) + : : +- BroadcastQueryStage (18), Statistics(X) + : : +- ColumnarBroadcastExchange (17) + : : +- ^ ProjectExecTransformer (15) + : : +- ^ NoopFilter (14) + : : +- ^ Scan parquet (13) + : +- ^ FilterExecTransformer (44) + : +- ^ ProjectExecTransformer (43) + : +- ^ RegularHashAggregateExecTransformer (42) + : +- ^ InputIteratorTransformer (41) + : +- ShuffleQueryStage (39), Statistics(X) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ FlushableHashAggregateExecTransformer (35) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (34) + : :- ^ ProjectExecTransformer (29) + : : +- ^ NoopFilter (28) + : : +- ^ Scan parquet (27) + : +- ^ InputIteratorTransformer (33) + : +- BroadcastQueryStage (31), Statistics(X) + : +- ReusedExchange (30) + +- ^ InputIteratorTransformer (61) + +- BroadcastQueryStage (59), Statistics(X) + +- ColumnarBroadcastExchange (58) + +- ^ ProjectExecTransformer (56) + +- ^ NoopFilter (55) + +- ^ Scan parquet (54) +- == Initial Plan == - Sort (103) - +- Exchange (102) - +- Project (101) - +- BroadcastHashJoin Inner BuildRight (100) - :- Project (95) - : +- ShuffledHashJoin LeftSemi BuildRight (94) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Exchange (93) - : +- Project (92) - : +- BroadcastHashJoin Inner BuildLeft (91) - : :- BroadcastExchange (78) - : : +- BroadcastHashJoin LeftSemi BuildRight (77) - : : :- Filter (72) - : : : +- Scan parquet (71) - : : +- BroadcastExchange (76) - : : +- Project (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- Filter (90) - : +- HashAggregate (89) - : +- Exchange (88) - : +- HashAggregate (87) - : +- BroadcastHashJoin LeftSemi BuildRight (86) - : :- Project (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (85) - : +- Project (84) - : +- Filter (83) - : +- Scan parquet (82) - +- BroadcastExchange (99) - +- Project (98) - +- Filter (97) - +- Scan parquet (96) + Sort (105) + +- Exchange (104) + +- Project (103) + +- BroadcastHashJoin Inner BuildRight (102) + :- Project (97) + : +- ShuffledHashJoin LeftSemi BuildRight (96) + : :- Exchange (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (95) + : +- Project (94) + : +- BroadcastHashJoin Inner BuildLeft (93) + : :- BroadcastExchange (80) + : : +- BroadcastHashJoin LeftSemi BuildRight (79) + : : :- Filter (74) + : : : +- Scan parquet (73) + : : +- BroadcastExchange (78) + : : +- Project (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- Filter (92) + : +- HashAggregate (91) + : +- Exchange (90) + : +- HashAggregate (89) + : +- BroadcastHashJoin LeftSemi BuildRight (88) + : :- Project (83) + : : +- Filter (82) + : : +- Scan parquet (81) + : +- BroadcastExchange (87) + : +- Project (86) + : +- Filter (85) + : +- Scan parquet (84) + +- BroadcastExchange (101) + +- Project (100) + +- Filter (99) + +- Scan parquet (98) (1) Scan parquet @@ -112,438 +114,446 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) AQEShuffleRead +(8) AQEShuffleRead Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: local -(8) InputAdapter +(9) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(10) Scan parquet +(11) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(11) NoopFilter +(12) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) Scan parquet +(13) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(13) NoopFilter +(14) NoopFilter Input [2]: [p_partkey#X, p_name#X] Arguments: [p_partkey#X, p_name#X] -(14) ProjectExecTransformer +(15) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(15) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(16) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(17) BroadcastQueryStage +(18) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [1]: [p_partkey#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [1]: [p_partkey#X] -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(26) Scan parquet +(27) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(27) NoopFilter +(28) NoopFilter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(29) ReusedExchange [Reuses operator id: 16] +(30) ReusedExchange [Reuses operator id: 17] Output [1]: [p_partkey#X] -(30) BroadcastQueryStage +(31) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [1]: [p_partkey#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [1]: [p_partkey#X] -(33) BroadcastHashJoinExecTransformer +(34) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(34) FlushableHashAggregateExecTransformer +(35) FlushableHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(36) ProjectExecTransformer Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(38) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(39) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(42) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) FilterExecTransformer +(44) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(44) BroadcastHashJoinExecTransformer +(45) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(45) ProjectExecTransformer +(46) ProjectExecTransformer Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(46) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] Arguments: false -(47) ColumnarBroadcastExchange +(48) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(48) BroadcastQueryStage +(49) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(49) InputAdapter +(50) InputAdapter Input [1]: [ps_suppkey#X] -(50) InputIteratorTransformer +(51) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(51) BroadcastHashJoinExecTransformer +(52) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(52) ProjectExecTransformer +(53) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(53) Scan parquet +(54) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(54) NoopFilter +(55) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(55) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(56) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(57) ColumnarBroadcastExchange +(58) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(58) BroadcastQueryStage +(59) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(59) InputAdapter +(60) InputAdapter Input [1]: [n_nationkey#X] -(60) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [n_nationkey#X] -(61) BroadcastHashJoinExecTransformer +(62) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) ProjectExecTransformer +(63) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(63) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(64) ColumnarExchange +(65) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(66) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(65) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(66) AQEShuffleRead +(68) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(67) VeloxColumnarToRowExec +(69) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(68) Scan parquet +(70) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(69) Filter +(71) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(70) Exchange +(72) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(73) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(72) Filter +(74) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(73) Scan parquet +(75) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(74) Filter +(76) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(75) Project +(77) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(76) BroadcastExchange +(78) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(77) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(78) BroadcastExchange +(80) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(79) Scan parquet +(81) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(80) Filter +(82) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(81) Project +(83) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(82) Scan parquet +(84) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(83) Filter +(85) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(84) Project +(86) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(85) BroadcastExchange +(87) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(86) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(87) HashAggregate +(89) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(88) Exchange +(90) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(91) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(90) Filter +(92) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(91) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(92) Project +(94) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(93) Exchange +(95) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(95) Project +(97) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(96) Scan parquet +(98) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(97) Filter +(99) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(98) Project +(100) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(99) BroadcastExchange +(101) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(100) BroadcastHashJoin +(102) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(101) Project +(103) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(102) Exchange +(104) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) Sort +(105) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(104) AdaptiveSparkPlan +(106) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt index c52c51139d2a..9f48891225fb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt @@ -1,84 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (91) +AdaptiveSparkPlan (92) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- BroadcastQueryStage (5), Statistics(X) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ NoopFilter (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5), Statistics(X) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ NoopFilter (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (90) - +- HashAggregate (89) - +- Exchange (88) - +- HashAggregate (87) - +- Project (86) - +- BroadcastHashJoin Inner BuildRight (85) - :- Project (80) - : +- BroadcastHashJoin Inner BuildRight (79) - : :- Project (74) - : : +- BroadcastHashJoin Inner BuildLeft (73) - : : :- BroadcastExchange (61) - : : : +- Filter (60) - : : : +- Scan parquet (59) - : : +- BroadcastHashJoin LeftAnti BuildRight (72) - : : :- BroadcastHashJoin LeftSemi BuildRight (67) - : : : :- Project (64) - : : : : +- Filter (63) - : : : : +- Scan parquet (62) - : : : +- BroadcastExchange (66) - : : : +- Scan parquet (65) - : : +- BroadcastExchange (71) - : : +- Project (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- BroadcastExchange (78) - : +- Project (77) - : +- Filter (76) - : +- Scan parquet (75) - +- BroadcastExchange (84) - +- Project (83) - +- Filter (82) - +- Scan parquet (81) + TakeOrderedAndProject (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (81) + : +- BroadcastHashJoin Inner BuildRight (80) + : :- Project (75) + : : +- BroadcastHashJoin Inner BuildLeft (74) + : : :- BroadcastExchange (62) + : : : +- Filter (61) + : : : +- Scan parquet (60) + : : +- BroadcastHashJoin LeftAnti BuildRight (73) + : : :- BroadcastHashJoin LeftSemi BuildRight (68) + : : : :- Project (65) + : : : : +- Filter (64) + : : : : +- Scan parquet (63) + : : : +- BroadcastExchange (67) + : : : +- Scan parquet (66) + : : +- BroadcastExchange (72) + : : +- Project (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- BroadcastExchange (79) + : +- Project (78) + : +- Filter (77) + : +- Scan parquet (76) + +- BroadcastExchange (85) + +- Project (84) + +- Filter (83) + +- Scan parquet (82) (1) Scan parquet @@ -285,7 +286,7 @@ Join condition: None Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(49) FlushableHashAggregateExecTransformer +(49) RegularHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] @@ -300,190 +301,194 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(53) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [2]: [s_name#X, count#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(58) VeloxColumnarToRowExec +(59) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(59) Scan parquet +(60) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(60) Filter +(61) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(61) BroadcastExchange +(62) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(62) Scan parquet +(63) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(63) Filter +(64) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(64) Project +(65) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(66) BroadcastExchange +(67) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(67) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(68) Scan parquet +(69) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(69) Filter +(70) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(70) Project +(71) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(71) BroadcastExchange +(72) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(74) Project +(75) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(75) Scan parquet +(76) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(76) Filter +(77) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(77) Project +(78) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(78) BroadcastExchange +(79) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(79) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(80) Project +(81) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(81) Scan parquet +(82) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(82) Filter +(83) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(83) Project +(84) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(84) BroadcastExchange +(85) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(85) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(86) Project +(87) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(87) HashAggregate +(88) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(88) Exchange +(89) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(90) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(90) TakeOrderedAndProject +(91) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(91) AdaptiveSparkPlan +(92) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt index d404e2149a89..404a3debc42c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (39) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (27) + +- ^ SortExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- VeloxAppendBatches (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (37) - +- Exchange (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- BroadcastHashJoin LeftAnti BuildRight (31) - :- Filter (28) - : +- Scan parquet (27) - +- BroadcastExchange (30) - +- Scan parquet (29) + Sort (38) + +- Exchange (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- BroadcastHashJoin LeftAnti BuildRight (32) + :- Filter (29) + : +- Scan parquet (28) + +- BroadcastExchange (31) + +- Scan parquet (30) (1) Scan parquet @@ -118,201 +119,205 @@ Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacc Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) ColumnarExchange +(20) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(21) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(22) InputAdapter +(23) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) SortExecTransformer +(25) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(27) Scan parquet +(28) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(28) Filter +(29) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(29) Scan parquet +(30) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(30) BroadcastExchange +(31) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(31) BroadcastHashJoin +(32) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(32) Project +(33) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(33) HashAggregate +(34) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(34) Exchange +(35) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(36) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(36) Exchange +(37) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Sort +(38) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(38) AdaptiveSparkPlan +(39) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (58) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ RegularHashAggregateExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (51) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ InputIteratorTransformer (48) + +- ShuffleQueryStage (46), Statistics(X) + +- ColumnarExchange (45) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) + HashAggregate (57) + +- Exchange (56) + +- HashAggregate (55) + +- Project (54) + +- Filter (53) + +- Scan parquet (52) -(39) Scan parquet +(40) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(40) NoopFilter +(41) NoopFilter Input [2]: [c_phone#X, c_acctbal#X] Arguments: [c_phone#X, c_acctbal#X] -(41) ProjectExecTransformer +(42) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(42) FlushableHashAggregateExecTransformer +(43) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(43) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(44) ColumnarExchange +(45) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(46) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(46) InputAdapter +(47) InputAdapter Input [2]: [sum#X, count#X] -(47) InputIteratorTransformer +(48) InputIteratorTransformer Input [2]: [sum#X, count#X] -(48) RegularHashAggregateExecTransformer +(49) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(49) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(50) VeloxColumnarToRowExec +(51) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(51) Scan parquet +(52) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(52) Filter +(53) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(53) Project +(54) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(54) HashAggregate +(55) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(55) Exchange +(56) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(57) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) AdaptiveSparkPlan +(58) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt index dbe67ba34d04..aacc67b90b3f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- TakeOrderedAndProjectExecTransformer (33) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- BroadcastQueryStage (18), Statistics(X) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ NoopFilter (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (35) + +- TakeOrderedAndProjectExecTransformer (34) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ RegularHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ NoopFilter (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildLeft (41) - : :- BroadcastExchange (38) - : : +- Project (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + TakeOrderedAndProject (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- BroadcastHashJoin Inner BuildRight (48) + :- Project (43) + : +- BroadcastHashJoin Inner BuildLeft (42) + : :- BroadcastExchange (39) + : : +- Project (38) + : : +- Filter (37) + : : +- Scan parquet (36) + : +- Filter (41) + : +- Scan parquet (40) + +- BroadcastExchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -144,7 +145,7 @@ Join condition: None Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) FlushableHashAggregateExecTransformer +(23) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -159,131 +160,135 @@ Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: X + +(27) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(33) TakeOrderedAndProjectExecTransformer +(34) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(34) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(35) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(36) Filter +(37) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(37) Project +(38) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(38) BroadcastExchange +(39) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) Scan parquet +(40) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(42) Project +(43) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(43) Scan parquet +(44) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(45) Project +(46) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) BroadcastExchange +(47) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(48) Project +(49) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(49) HashAggregate +(50) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(50) Exchange +(51) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(52) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(52) TakeOrderedAndProject +(53) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt index 626f608cecd0..5f3ea7038ab0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt @@ -1,43 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9), Statistics(X) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- BroadcastHashJoin LeftSemi BuildRight (37) - :- Project (32) - : +- Filter (31) - : +- Scan parquet (30) - +- BroadcastExchange (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (44) + +- Exchange (43) + +- HashAggregate (42) + +- Exchange (41) + +- HashAggregate (40) + +- Project (39) + +- BroadcastHashJoin LeftSemi BuildRight (38) + :- Project (33) + : +- Filter (32) + : +- Scan parquet (31) + +- BroadcastExchange (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -137,100 +138,104 @@ Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(27) SortExecTransformer +(28) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(30) Scan parquet +(31) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(31) Filter +(32) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(32) Project +(33) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(33) Scan parquet +(34) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(34) Filter +(35) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(35) Project +(36) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(36) BroadcastExchange +(37) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(37) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(38) Project +(39) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(39) HashAggregate +(40) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(40) Exchange +(41) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) HashAggregate +(42) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(42) Exchange +(43) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(44) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt index 7a2ce7cf7a4a..d83f2507a100 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt @@ -1,91 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (100) +AdaptiveSparkPlan (101) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45), Statistics(X) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61), Statistics(X) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (99) - +- Exchange (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Project (71) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (92) - +- Project (91) - +- Filter (90) - +- Scan parquet (89) + Sort (100) + +- Exchange (99) + +- HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- BroadcastHashJoin Inner BuildRight (94) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Project (72) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (93) + +- Project (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -337,196 +338,200 @@ Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedpric Input [2]: [n_name#X, revenue#X] Arguments: false -(59) ColumnarExchange +(59) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(60) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(61) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(61) InputAdapter +(62) InputAdapter Input [2]: [n_name#X, revenue#X] -(62) InputIteratorTransformer +(63) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(63) SortExecTransformer +(64) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(65) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(67) Filter +(68) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(68) BroadcastExchange +(69) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(71) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(73) Project +(74) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(75) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Project +(79) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(79) Scan parquet +(80) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(81) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(81) BroadcastExchange +(82) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(83) Project +(84) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(84) Scan parquet +(85) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(86) BroadcastExchange +(87) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) Project +(89) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(89) Scan parquet +(90) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(91) Project +(92) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(92) BroadcastExchange +(93) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(93) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(94) Project +(95) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(95) HashAggregate +(96) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(96) Exchange +(97) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(98) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(98) Exchange +(99) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Sort +(100) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(100) AdaptiveSparkPlan +(101) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt index e03830992c2e..30e1b3fe013b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt index 4f9b5f20956d..24cd98650a17 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt @@ -1,85 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ SortExecTransformer (58) - +- ^ InputIteratorTransformer (57) - +- ShuffleQueryStage (55), Statistics(X) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16), Statistics(X) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ NoopFilter (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25), Statistics(X) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ NoopFilter (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ NoopFilter (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40), Statistics(X) - +- ReusedExchange (39) + VeloxColumnarToRowExec (61) + +- ^ SortExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- VeloxAppendBatches (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ NoopFilter (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ NoopFilter (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == - Sort (92) - +- Exchange (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (63) - : : : : : +- Filter (62) - : : : : : +- Scan parquet (61) - : : : : +- Filter (65) - : : : : +- Scan parquet (64) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (85) - +- Filter (84) - +- Scan parquet (83) + Sort (93) + +- Exchange (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- BroadcastHashJoin Inner BuildRight (87) + :- Project (83) + : +- BroadcastHashJoin Inner BuildRight (82) + : :- Project (78) + : : +- BroadcastHashJoin Inner BuildRight (77) + : : :- Project (73) + : : : +- BroadcastHashJoin Inner BuildRight (72) + : : : :- Project (68) + : : : : +- BroadcastHashJoin Inner BuildLeft (67) + : : : : :- BroadcastExchange (64) + : : : : : +- Filter (63) + : : : : : +- Scan parquet (62) + : : : : +- Filter (66) + : : : : +- Scan parquet (65) + : : : +- BroadcastExchange (71) + : : : +- Filter (70) + : : : +- Scan parquet (69) + : : +- BroadcastExchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- BroadcastExchange (81) + : +- Filter (80) + : +- Scan parquet (79) + +- BroadcastExchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -307,188 +308,192 @@ Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) ColumnarExchange +(54) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(55) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(56) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(56) InputAdapter +(57) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(57) InputIteratorTransformer +(58) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) SortExecTransformer +(59) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(59) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(60) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(61) Scan parquet +(62) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(63) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(63) BroadcastExchange +(64) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(64) Scan parquet +(65) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(65) Filter +(66) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(66) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(67) Project +(68) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(68) Scan parquet +(69) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(69) Filter +(70) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(70) BroadcastExchange +(71) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(72) Project +(73) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(77) Project +(78) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(79) Filter +(80) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(80) BroadcastExchange +(81) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(83) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(87) Project +(88) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(88) HashAggregate +(89) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(89) Exchange +(90) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(91) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(91) Exchange +(92) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(93) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(93) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt index 677a42eb711a..8185bc4a349e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt @@ -1,116 +1,117 @@ == Physical Plan == -AdaptiveSparkPlan (129) +AdaptiveSparkPlan (130) +- == Final Plan == - VeloxColumnarToRowExec (84) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ NoopFilter (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26), Statistics(X) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ NoopFilter (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35), Statistics(X) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ NoopFilter (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44), Statistics(X) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ NoopFilter (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53), Statistics(X) - : +- ColumnarBroadcastExchange (52) - : +- ^ NoopFilter (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63), Statistics(X) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ NoopFilter (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (85) + +- ^ SortExecTransformer (83) + +- ^ InputIteratorTransformer (82) + +- ShuffleQueryStage (80), Statistics(X) + +- ColumnarExchange (79) + +- VeloxAppendBatches (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ShuffleQueryStage (72), Statistics(X) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ NoopFilter (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ NoopFilter (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ NoopFilter (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ NoopFilter (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ NoopFilter (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ NoopFilter (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (128) - +- Exchange (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- BroadcastHashJoin Inner BuildRight (122) - :- Project (117) - : +- BroadcastHashJoin Inner BuildRight (116) - : :- Project (112) - : : +- BroadcastHashJoin Inner BuildRight (111) - : : :- Project (107) - : : : +- BroadcastHashJoin Inner BuildRight (106) - : : : :- Project (102) - : : : : +- BroadcastHashJoin Inner BuildRight (101) - : : : : :- Project (97) - : : : : : +- BroadcastHashJoin Inner BuildRight (96) - : : : : : :- Project (92) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) - : : : : : : :- BroadcastExchange (88) - : : : : : : : +- Project (87) - : : : : : : : +- Filter (86) - : : : : : : : +- Scan parquet (85) - : : : : : : +- Filter (90) - : : : : : : +- Scan parquet (89) - : : : : : +- BroadcastExchange (95) - : : : : : +- Filter (94) - : : : : : +- Scan parquet (93) - : : : : +- BroadcastExchange (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- BroadcastExchange (105) - : : : +- Filter (104) - : : : +- Scan parquet (103) - : : +- BroadcastExchange (110) - : : +- Filter (109) - : : +- Scan parquet (108) - : +- BroadcastExchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- BroadcastExchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (129) + +- Exchange (128) + +- HashAggregate (127) + +- Exchange (126) + +- HashAggregate (125) + +- Project (124) + +- BroadcastHashJoin Inner BuildRight (123) + :- Project (118) + : +- BroadcastHashJoin Inner BuildRight (117) + : :- Project (113) + : : +- BroadcastHashJoin Inner BuildRight (112) + : : :- Project (108) + : : : +- BroadcastHashJoin Inner BuildRight (107) + : : : :- Project (103) + : : : : +- BroadcastHashJoin Inner BuildRight (102) + : : : : :- Project (98) + : : : : : +- BroadcastHashJoin Inner BuildRight (97) + : : : : : :- Project (93) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (92) + : : : : : : :- BroadcastExchange (89) + : : : : : : : +- Project (88) + : : : : : : : +- Filter (87) + : : : : : : : +- Scan parquet (86) + : : : : : : +- Filter (91) + : : : : : : +- Scan parquet (90) + : : : : : +- BroadcastExchange (96) + : : : : : +- Filter (95) + : : : : : +- Scan parquet (94) + : : : : +- BroadcastExchange (101) + : : : : +- Filter (100) + : : : : +- Scan parquet (99) + : : : +- BroadcastExchange (106) + : : : +- Filter (105) + : : : +- Scan parquet (104) + : : +- BroadcastExchange (111) + : : +- Filter (110) + : : +- Scan parquet (109) + : +- BroadcastExchange (116) + : +- Filter (115) + : +- Scan parquet (114) + +- BroadcastExchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) (1) Scan parquet @@ -442,244 +443,248 @@ Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.000 Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) ColumnarExchange +(78) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(79) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(80) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(80) InputAdapter +(81) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(81) InputIteratorTransformer +(82) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(82) SortExecTransformer +(83) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(83) WholeStageCodegenTransformer (X) +(84) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(84) VeloxColumnarToRowExec +(85) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(85) Scan parquet +(86) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(87) Project +(88) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(88) BroadcastExchange +(89) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) Scan parquet +(90) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(92) Project +(93) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(93) Scan parquet +(94) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(94) Filter +(95) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(95) BroadcastExchange +(96) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(96) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(97) Project +(98) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(98) Scan parquet +(99) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(99) Filter +(100) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(100) BroadcastExchange +(101) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(101) BroadcastHashJoin +(102) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(102) Project +(103) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(103) Scan parquet +(104) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(104) Filter +(105) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(105) BroadcastExchange +(106) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(106) BroadcastHashJoin +(107) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(107) Project +(108) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(108) Scan parquet +(109) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(109) Filter +(110) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(110) BroadcastExchange +(111) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(111) BroadcastHashJoin +(112) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(112) Project +(113) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(113) Scan parquet +(114) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(114) Filter +(115) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(115) BroadcastExchange +(116) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(116) BroadcastHashJoin +(117) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(117) Project +(118) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(118) Scan parquet +(119) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(119) Filter +(120) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(120) Project +(121) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(121) BroadcastExchange +(122) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(122) BroadcastHashJoin +(123) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(123) Project +(124) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(124) HashAggregate +(125) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) Exchange +(126) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(127) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(127) Exchange +(128) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(129) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(129) AdaptiveSparkPlan +(130) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt index 9f2111b015ca..0d27f28fda4d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt @@ -1,89 +1,90 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (99) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- ^ SortExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (65) + +- ^ SortExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (97) - +- Exchange (96) - +- HashAggregate (95) - +- Exchange (94) - +- HashAggregate (93) - +- Project (92) - +- BroadcastHashJoin Inner BuildRight (91) - :- Project (87) - : +- BroadcastHashJoin Inner BuildRight (86) - : :- Project (82) - : : +- BroadcastHashJoin Inner BuildRight (81) - : : :- Project (77) - : : : +- BroadcastHashJoin Inner BuildRight (76) - : : : :- Project (72) - : : : : +- BroadcastHashJoin Inner BuildLeft (71) - : : : : :- BroadcastExchange (68) - : : : : : +- Project (67) - : : : : : +- Filter (66) - : : : : : +- Scan parquet (65) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (75) - : : : +- Filter (74) - : : : +- Scan parquet (73) - : : +- BroadcastExchange (80) - : : +- Filter (79) - : : +- Scan parquet (78) - : +- BroadcastExchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- BroadcastExchange (90) - +- Filter (89) - +- Scan parquet (88) + Sort (98) + +- Exchange (97) + +- HashAggregate (96) + +- Exchange (95) + +- HashAggregate (94) + +- Project (93) + +- BroadcastHashJoin Inner BuildRight (92) + :- Project (88) + : +- BroadcastHashJoin Inner BuildRight (87) + : :- Project (83) + : : +- BroadcastHashJoin Inner BuildRight (82) + : : :- Project (78) + : : : +- BroadcastHashJoin Inner BuildRight (77) + : : : :- Project (73) + : : : : +- BroadcastHashJoin Inner BuildLeft (72) + : : : : :- BroadcastExchange (69) + : : : : : +- Project (68) + : : : : : +- Filter (67) + : : : : : +- Scan parquet (66) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (76) + : : : +- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- BroadcastExchange (86) + : +- Filter (85) + : +- Scan parquet (84) + +- BroadcastExchange (91) + +- Filter (90) + +- Scan parquet (89) (1) Scan parquet @@ -331,192 +332,196 @@ Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) ColumnarExchange +(58) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(59) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(60) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(60) InputAdapter +(61) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(61) InputIteratorTransformer +(62) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) SortExecTransformer +(63) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(63) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(64) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(67) Project +(68) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(69) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(72) Project +(73) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(77) Project +(78) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(79) Filter +(80) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(80) BroadcastExchange +(81) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(82) Project +(83) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(87) Project +(88) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(88) Scan parquet +(89) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(90) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(90) BroadcastExchange +(91) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(92) Project +(93) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(93) HashAggregate +(94) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(94) Exchange +(95) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) HashAggregate +(96) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(96) Exchange +(97) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(98) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(98) AdaptiveSparkPlan +(99) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt index c254ec8c82ca..589019c4cdbf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt index ffc37f5b7908..f83547f843ca 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt @@ -1,64 +1,65 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- BroadcastQueryStage (8), Statistics(X) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ NoopFilter (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- BroadcastQueryStage (18), Statistics(X) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ NoopFilter (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- BroadcastQueryStage (27), Statistics(X) - +- ColumnarBroadcastExchange (26) - +- ^ NoopFilter (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (44) + +- TakeOrderedAndProjectExecTransformer (43) + +- ^ ProjectExecTransformer (41) + +- ^ RegularHashAggregateExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- VeloxAppendBatches (35) + +- ^ ProjectExecTransformer (33) + +- ^ RegularHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ NoopFilter (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ NoopFilter (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ NoopFilter (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- BroadcastHashJoin Inner BuildRight (61) - :- Project (57) - : +- BroadcastHashJoin Inner BuildRight (56) - : :- Project (51) - : : +- BroadcastHashJoin Inner BuildRight (50) - : : :- Filter (45) - : : : +- Scan parquet (44) - : : +- BroadcastExchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (55) - : +- Project (54) - : +- Filter (53) - : +- Scan parquet (52) - +- BroadcastExchange (60) - +- Filter (59) - +- Scan parquet (58) + TakeOrderedAndProject (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- BroadcastHashJoin Inner BuildRight (62) + :- Project (58) + : +- BroadcastHashJoin Inner BuildRight (57) + : :- Project (52) + : : +- BroadcastHashJoin Inner BuildRight (51) + : : :- Filter (46) + : : : +- Scan parquet (45) + : : +- BroadcastExchange (50) + : : +- Project (49) + : : +- Filter (48) + : : +- Scan parquet (47) + : +- BroadcastExchange (56) + : +- Project (55) + : +- Filter (54) + : +- Scan parquet (53) + +- BroadcastExchange (61) + +- Filter (60) + +- Scan parquet (59) (1) Scan parquet @@ -197,7 +198,7 @@ Join condition: None Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(32) FlushableHashAggregateExecTransformer +(32) RegularHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -212,158 +213,162 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(35) ColumnarExchange +(35) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(36) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(37) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(37) InputAdapter +(38) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(38) InputIteratorTransformer +(39) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(40) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(40) ProjectExecTransformer +(41) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(41) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(43) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(43) VeloxColumnarToRowExec +(44) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(44) Scan parquet +(45) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(46) Scan parquet +(47) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(48) Project +(49) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(51) Project +(52) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(52) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(53) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(54) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(55) BroadcastExchange +(56) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(57) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(58) Scan parquet +(59) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(59) Filter +(60) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(60) BroadcastExchange +(61) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(61) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(62) Project +(63) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(63) HashAggregate +(64) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(64) Exchange +(65) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(66) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(66) TakeOrderedAndProject +(67) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt index 6410bb3fed5f..da2efbcb3134 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt @@ -1,55 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7), Statistics(X) - : +- ColumnarBroadcastExchange (6) - : +- ^ NoopFilter (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17), Statistics(X) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ NoopFilter (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ NoopFilter (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ NoopFilter (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (57) - +- Exchange (56) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- BroadcastHashJoin Inner BuildRight (50) - :- Project (45) - : +- BroadcastHashJoin Inner BuildRight (44) - : :- Filter (40) - : : +- Scan parquet (39) - : +- BroadcastExchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- BroadcastExchange (49) - +- Project (48) - +- Filter (47) - +- Scan parquet (46) + Sort (58) + +- Exchange (57) + +- Filter (56) + +- HashAggregate (55) + +- Exchange (54) + +- HashAggregate (53) + +- Project (52) + +- BroadcastHashJoin Inner BuildRight (51) + :- Project (46) + : +- BroadcastHashJoin Inner BuildRight (45) + : :- Filter (41) + : : +- Scan parquet (40) + : +- BroadcastExchange (44) + : +- Filter (43) + : +- Scan parquet (42) + +- BroadcastExchange (50) + +- Project (49) + +- Filter (48) + +- Scan parquet (47) (1) Scan parquet @@ -189,356 +190,360 @@ Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery su Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(34) InputAdapter +(35) InputAdapter Input [2]: [ps_partkey#X, value#X] -(35) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(36) SortExecTransformer +(37) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(38) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(39) Scan parquet +(40) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(41) Scan parquet +(42) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(42) Filter +(43) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(43) BroadcastExchange +(44) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(45) Project +(46) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(46) Scan parquet +(47) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(48) Project +(49) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(51) Project +(52) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(52) HashAggregate +(53) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(53) Exchange +(54) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(55) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(55) Filter +(56) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(56) Exchange +(57) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(58) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(58) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 30 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (99) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (82) - +- ^ ProjectExecTransformer (80) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) - :- ^ ProjectExecTransformer (66) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) - : :- ^ NoopFilter (60) - : : +- ^ Scan parquet (59) - : +- ^ InputIteratorTransformer (64) - : +- BroadcastQueryStage (62), Statistics(X) - : +- ReusedExchange (61) - +- ^ InputIteratorTransformer (70) - +- BroadcastQueryStage (68), Statistics(X) - +- ReusedExchange (67) + VeloxColumnarToRowExec (83) + +- ^ ProjectExecTransformer (81) + +- ^ RegularHashAggregateExecTransformer (80) + +- ^ InputIteratorTransformer (79) + +- ShuffleQueryStage (77), Statistics(X) + +- ColumnarExchange (76) + +- ^ FlushableHashAggregateExecTransformer (74) + +- ^ ProjectExecTransformer (73) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (72) + :- ^ ProjectExecTransformer (67) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + : :- ^ NoopFilter (61) + : : +- ^ Scan parquet (60) + : +- ^ InputIteratorTransformer (65) + : +- BroadcastQueryStage (63), Statistics(X) + : +- ReusedExchange (62) + +- ^ InputIteratorTransformer (71) + +- BroadcastQueryStage (69), Statistics(X) + +- ReusedExchange (68) +- == Initial Plan == - HashAggregate (98) - +- Exchange (97) - +- HashAggregate (96) - +- Project (95) - +- BroadcastHashJoin Inner BuildRight (94) - :- Project (89) - : +- BroadcastHashJoin Inner BuildRight (88) - : :- Filter (84) - : : +- Scan parquet (83) - : +- BroadcastExchange (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (93) - +- Project (92) - +- Filter (91) - +- Scan parquet (90) - - -(59) Scan parquet + HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Project (96) + +- BroadcastHashJoin Inner BuildRight (95) + :- Project (90) + : +- BroadcastHashJoin Inner BuildRight (89) + : :- Filter (85) + : : +- Scan parquet (84) + : +- BroadcastExchange (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (94) + +- Project (93) + +- Filter (92) + +- Scan parquet (91) + + +(60) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(60) NoopFilter +(61) NoopFilter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(61) ReusedExchange [Reuses operator id: 6] +(62) ReusedExchange [Reuses operator id: 6] Output [2]: [s_suppkey#X, s_nationkey#X] -(62) BroadcastQueryStage +(63) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(63) InputAdapter +(64) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(64) InputIteratorTransformer +(65) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(65) BroadcastHashJoinExecTransformer +(66) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(66) ProjectExecTransformer +(67) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(67) ReusedExchange [Reuses operator id: 16] +(68) ReusedExchange [Reuses operator id: 16] Output [1]: [n_nationkey#X] -(68) BroadcastQueryStage +(69) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(69) InputAdapter +(70) InputAdapter Input [1]: [n_nationkey#X] -(70) InputIteratorTransformer +(71) InputIteratorTransformer Input [1]: [n_nationkey#X] -(71) BroadcastHashJoinExecTransformer +(72) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(72) ProjectExecTransformer +(73) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(73) FlushableHashAggregateExecTransformer +(74) FlushableHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(74) WholeStageCodegenTransformer (X) +(75) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(75) ColumnarExchange +(76) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(77) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(77) InputAdapter +(78) InputAdapter Input [2]: [sum#X, isEmpty#X] -(78) InputIteratorTransformer +(79) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(79) RegularHashAggregateExecTransformer +(80) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(80) ProjectExecTransformer +(81) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(81) WholeStageCodegenTransformer (X) +(82) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(82) VeloxColumnarToRowExec +(83) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(83) Scan parquet +(84) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(84) Filter +(85) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(85) Scan parquet +(86) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(87) BroadcastExchange +(88) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(89) Project +(90) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(90) Scan parquet +(91) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(91) Filter +(92) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(92) Project +(93) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(93) BroadcastExchange +(94) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(94) BroadcastHashJoin +(95) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(95) Project +(96) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(96) HashAggregate +(97) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(97) Exchange +(98) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(98) HashAggregate +(99) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(99) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt index 4e85d8194098..6006564bf2a8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt @@ -1,41 +1,42 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (43) +- == Final Plan == - VeloxColumnarToRowExec (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (29) + +- ^ SortExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- VeloxAppendBatches (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) + Sort (42) + +- Exchange (41) + +- HashAggregate (40) + +- Exchange (39) + +- HashAggregate (38) + +- Project (37) + +- BroadcastHashJoin Inner BuildLeft (36) + :- BroadcastExchange (32) + : +- Filter (31) + : +- Scan parquet (30) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -132,97 +133,101 @@ Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_ Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) SortExecTransformer +(27) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(27) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(30) Filter +(31) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(31) BroadcastExchange +(32) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(32) Scan parquet +(33) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(33) Filter +(34) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(34) Project +(35) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(35) BroadcastHashJoin +(36) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(36) Project +(37) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(37) HashAggregate +(38) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(38) Exchange +(39) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(40) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(40) Exchange +(41) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(42) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(43) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt index e83ab1aca83c..434a767c15a4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt @@ -1,49 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ NoopFilter (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ NoopFilter (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- BroadcastHashJoin LeftOuter BuildRight (42) - :- Scan parquet (37) - +- BroadcastExchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- BroadcastHashJoin LeftOuter BuildRight (43) + :- Scan parquet (38) + +- BroadcastExchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -175,110 +176,114 @@ Results [2]: [c_count#X, count(1)#X AS custdist#X] Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(33) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(35) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(38) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(43) Project +(44) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) Exchange +(46) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(48) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(49) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(50) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(51) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(52) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt index 58d532222af5..e4a7a29b15b0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt @@ -1,35 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- BroadcastHashJoin Inner BuildRight (29) - :- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- BroadcastExchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- BroadcastHashJoin Inner BuildRight (30) + :- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- BroadcastExchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -86,7 +87,7 @@ Join condition: None Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] @@ -97,96 +98,100 @@ Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(23) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(25) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) Scan parquet +(27) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(28) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) BroadcastExchange +(29) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastHashJoin +(30) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(30) Project +(31) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(32) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) Exchange +(33) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] -(34) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt index 269921602469..f088b241bdd5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt @@ -1,42 +1,43 @@ == Physical Plan == -AdaptiveSparkPlan (41) +AdaptiveSparkPlan (42) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- AQEShuffleRead (25) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (27) + +- AQEShuffleRead (26) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (40) - +- Exchange (39) - +- Project (38) - +- BroadcastHashJoin Inner BuildLeft (37) - :- BroadcastExchange (29) - : +- Filter (28) - : +- Scan parquet (27) - +- Filter (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- Filter (31) - +- Scan parquet (30) + Sort (41) + +- Exchange (40) + +- Project (39) + +- BroadcastHashJoin Inner BuildLeft (38) + :- BroadcastExchange (30) + : +- Filter (29) + : +- Scan parquet (28) + +- Filter (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- Filter (32) + +- Scan parquet (31) (1) Scan parquet @@ -137,247 +138,251 @@ Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_ Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(25) AQEShuffleRead +(26) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(26) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) Scan parquet +(28) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(28) Filter +(29) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) Scan parquet +(31) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(32) Project +(33) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(33) HashAggregate +(34) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(34) Exchange +(35) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(36) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(36) Filter +(37) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(37) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(38) Project +(39) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(39) Exchange +(40) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) Sort +(41) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(41) AdaptiveSparkPlan +(42) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 19 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (66) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (57) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ ProjectExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ NoopFilter (43) - +- ^ Scan parquet (42) + VeloxColumnarToRowExec (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- ^ ProjectExecTransformer (47) + +- ^ FlushableHashAggregateExecTransformer (46) + +- ^ ProjectExecTransformer (45) + +- ^ NoopFilter (44) + +- ^ Scan parquet (43) +- == Initial Plan == - HashAggregate (65) - +- HashAggregate (64) - +- HashAggregate (63) - +- Exchange (62) - +- HashAggregate (61) - +- Project (60) - +- Filter (59) - +- Scan parquet (58) + HashAggregate (66) + +- HashAggregate (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- Project (61) + +- Filter (60) + +- Scan parquet (59) -(42) Scan parquet +(43) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(43) NoopFilter +(44) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(45) FlushableHashAggregateExecTransformer +(46) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(46) ProjectExecTransformer +(47) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(49) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(53) ProjectExecTransformer +(54) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(54) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(55) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(56) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(57) VeloxColumnarToRowExec +(58) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(58) Scan parquet +(59) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(59) Filter +(60) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(60) Project +(61) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(61) HashAggregate +(62) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(62) Exchange +(63) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(64) HashAggregate +(65) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(65) HashAggregate +(66) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(66) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt index 0a8771619020..3fff24e2b497 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt @@ -1,53 +1,54 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (57) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ NoopFilter (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ NoopFilter (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- BroadcastHashJoin Inner BuildRight (46) - :- BroadcastHashJoin LeftAnti BuildRight (42) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (45) - +- Filter (44) - +- Scan parquet (43) + Sort (56) + +- Exchange (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- BroadcastHashJoin Inner BuildRight (47) + :- BroadcastHashJoin LeftAnti BuildRight (43) + : :- Filter (38) + : : +- Scan parquet (37) + : +- BroadcastExchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- BroadcastExchange (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -176,136 +177,140 @@ Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_c Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) ColumnarExchange +(29) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(30) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(36) Scan parquet +(37) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(37) Filter +(38) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(38) Scan parquet +(39) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(39) Filter +(40) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(40) Project +(41) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: LeftAnti Join condition: None -(43) Scan parquet +(44) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(45) BroadcastExchange +(46) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(47) Project +(48) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(48) HashAggregate +(49) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(49) Exchange +(50) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(51) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(51) HashAggregate +(52) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(52) Exchange +(53) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(54) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(56) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(56) AdaptiveSparkPlan +(57) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/17.txt index 62612096db42..e499293269fc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/17.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (14) - +- ^ ProjectExecTransformer (12) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- RowToVeloxColumnar (2) - +- LocalTableScan (1) + VeloxColumnarToRowExec (15) + +- ^ ProjectExecTransformer (13) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ RegularHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == - HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin Inner BuildRight (30) - :- Project (22) - : +- BroadcastHashJoin Inner BuildRight (21) - : :- Filter (16) - : : +- Scan parquet (15) - : +- BroadcastExchange (20) - : +- Project (19) - : +- Filter (18) - : +- Scan parquet (17) - +- BroadcastExchange (29) - +- Filter (28) - +- HashAggregate (27) - +- Exchange (26) - +- HashAggregate (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin Inner BuildRight (31) + :- Project (23) + : +- BroadcastHashJoin Inner BuildRight (22) + : :- Filter (17) + : : +- Scan parquet (16) + : +- BroadcastExchange (21) + : +- Project (20) + : +- Filter (19) + : +- Scan parquet (18) + +- BroadcastExchange (30) + +- Filter (29) + +- HashAggregate (28) + +- Exchange (27) + +- HashAggregate (26) + +- Filter (25) + +- Scan parquet (24) (1) LocalTableScan @@ -47,7 +48,7 @@ Input [1]: [l_extendedprice#X] (4) InputIteratorTransformer Input [1]: [l_extendedprice#X] -(5) FlushableHashAggregateExecTransformer +(5) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] @@ -58,143 +59,147 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(8) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [2]: [sum#X, isEmpty#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(14) VeloxColumnarToRowExec +(15) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(15) Scan parquet +(16) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(16) Filter +(17) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(17) Scan parquet +(18) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(18) Filter +(19) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(19) Project +(20) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(20) BroadcastExchange +(21) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastHashJoin +(22) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(22) Project +(23) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(23) Scan parquet +(24) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(25) HashAggregate +(26) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(26) Exchange +(27) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) HashAggregate +(28) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(28) Filter +(29) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(31) Project +(32) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(32) HashAggregate +(33) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] -(35) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt index f0420b98194d..3408aba7dcbc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt @@ -1,80 +1,82 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- TakeOrderedAndProjectExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ NoopFilter (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23), Statistics(X) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- BroadcastQueryStage (32), Statistics(X) - +- ReusedExchange (31) + VeloxColumnarToRowExec (55) + +- TakeOrderedAndProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- VeloxAppendBatches (47) + +- ^ ProjectExecTransformer (45) + +- ^ RegularHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) + :- ^ ProjectExecTransformer (29) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (28) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (27) + : :- ^ NoopFilter (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (26) + : +- BroadcastQueryStage (24), Statistics(X) + : +- ColumnarBroadcastExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FilterExecTransformer (20) + : +- ^ RegularHashAggregateExecTransformer (19) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ RegularHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (41) + +- BroadcastQueryStage (39), Statistics(X) + +- ColumnarBroadcastExchange (38) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (36) + :- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (35) + +- BroadcastQueryStage (33), Statistics(X) + +- ReusedExchange (32) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (68) - : +- BroadcastHashJoin Inner BuildLeft (67) - : :- BroadcastExchange (56) - : : +- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastHashJoin LeftSemi BuildRight (66) - : :- Filter (58) - : : +- Scan parquet (57) - : +- BroadcastExchange (65) - : +- Project (64) - : +- Filter (63) - : +- HashAggregate (62) - : +- Exchange (61) - : +- HashAggregate (60) - : +- Scan parquet (59) - +- BroadcastExchange (79) - +- BroadcastHashJoin LeftSemi BuildRight (78) - :- Filter (70) - : +- Scan parquet (69) - +- BroadcastExchange (77) - +- Project (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Scan parquet (71) + TakeOrderedAndProject (87) + +- HashAggregate (86) + +- Exchange (85) + +- HashAggregate (84) + +- Project (83) + +- BroadcastHashJoin Inner BuildRight (82) + :- Project (70) + : +- BroadcastHashJoin Inner BuildLeft (69) + : :- BroadcastExchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- BroadcastHashJoin LeftSemi BuildRight (68) + : :- Filter (60) + : : +- Scan parquet (59) + : +- BroadcastExchange (67) + : +- Project (66) + : +- Filter (65) + : +- HashAggregate (64) + : +- Exchange (63) + : +- HashAggregate (62) + : +- Scan parquet (61) + +- BroadcastExchange (81) + +- BroadcastHashJoin LeftSemi BuildRight (80) + :- Filter (72) + : +- Scan parquet (71) + +- BroadcastExchange (79) + +- Project (78) + +- Filter (77) + +- HashAggregate (76) + +- Exchange (75) + +- HashAggregate (74) + +- Scan parquet (73) (1) Scan parquet @@ -123,7 +125,7 @@ Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) FlushableHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] @@ -138,341 +140,349 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [1]: [l_orderkey#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [1]: [l_orderkey#X] -(26) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(27) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(30) NoopFilter +(31) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(31) ReusedExchange [Reuses operator id: 22] +(32) ReusedExchange [Reuses operator id: 23] Output [1]: [l_orderkey#X] -(32) BroadcastQueryStage +(33) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [1]: [l_orderkey#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [l_orderkey#X] -(35) BroadcastHashJoinExecTransformer +(36) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(37) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(39) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(41) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(43) FlushableHashAggregateExecTransformer +(44) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(47) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: X + +(48) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(49) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(50) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(51) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(51) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(52) TakeOrderedAndProjectExecTransformer +(54) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(53) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(54) Scan parquet +(56) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(56) BroadcastExchange +(58) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(59) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(58) Filter +(60) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(59) Scan parquet +(61) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(60) HashAggregate +(62) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(61) Exchange +(63) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(63) Filter +(65) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(64) Project +(66) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(65) BroadcastExchange +(67) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(67) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(68) Project +(70) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(69) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(71) Scan parquet +(73) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(74) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(75) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(76) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(77) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(78) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) BroadcastExchange +(79) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(79) BroadcastExchange +(81) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(81) Project +(83) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(82) HashAggregate +(84) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(83) Exchange +(85) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(86) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(85) TakeOrderedAndProject +(87) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(86) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt index c39011756bca..7fa7f7108a17 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt @@ -1,34 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (32) - +- Exchange (31) - +- HashAggregate (30) - +- Project (29) - +- BroadcastHashJoin Inner BuildRight (28) - :- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- BroadcastExchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -85,7 +86,7 @@ Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -96,92 +97,96 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(20) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(22) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(23) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(24) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(25) Scan parquet +(26) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(27) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) BroadcastExchange +(28) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(30) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(31) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) Exchange +(32) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(32) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(33) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt index 6bad140301d8..a4be44f50e60 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt @@ -1,90 +1,91 @@ == Physical Plan == -AdaptiveSparkPlan (96) +AdaptiveSparkPlan (97) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- AQEShuffleRead (59) - +- ShuffleQueryStage (58), Statistics(X) - +- ColumnarExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (54) - :- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (43) - : +- BroadcastQueryStage (41), Statistics(X) - : +- ColumnarBroadcastExchange (40) - : +- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (37) - : :- ^ InputIteratorTransformer (18) - : : +- BroadcastQueryStage (16), Statistics(X) - : : +- ColumnarBroadcastExchange (15) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (13) - : : :- ^ NoopFilter (4) - : : : +- ^ Scan parquet (3) - : : +- ^ InputIteratorTransformer (12) - : : +- BroadcastQueryStage (10), Statistics(X) - : : +- ColumnarBroadcastExchange (9) - : : +- ^ ProjectExecTransformer (7) - : : +- ^ NoopFilter (6) - : : +- ^ Scan parquet (5) - : +- ^ FilterExecTransformer (36) - : +- ^ ProjectExecTransformer (35) - : +- ^ RegularHashAggregateExecTransformer (34) - : +- ^ InputIteratorTransformer (33) - : +- ShuffleQueryStage (31), Statistics(X) - : +- ColumnarExchange (30) - : +- ^ ProjectExecTransformer (28) - : +- ^ FlushableHashAggregateExecTransformer (27) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ ProjectExecTransformer (21) - : : +- ^ NoopFilter (20) - : : +- ^ Scan parquet (19) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23), Statistics(X) - : +- ReusedExchange (22) - +- ^ InputIteratorTransformer (53) - +- BroadcastQueryStage (51), Statistics(X) - +- ColumnarBroadcastExchange (50) - +- ^ ProjectExecTransformer (48) - +- ^ NoopFilter (47) - +- ^ Scan parquet (46) + VeloxColumnarToRowExec (61) + +- AQEShuffleRead (60) + +- ShuffleQueryStage (59), Statistics(X) + +- ColumnarExchange (58) + +- VeloxAppendBatches (57) + +- ^ ProjectExecTransformer (55) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (54) + :- ^ ProjectExecTransformer (45) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (44) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (43) + : +- BroadcastQueryStage (41), Statistics(X) + : +- ColumnarBroadcastExchange (40) + : +- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (37) + : :- ^ InputIteratorTransformer (18) + : : +- BroadcastQueryStage (16), Statistics(X) + : : +- ColumnarBroadcastExchange (15) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (13) + : : :- ^ NoopFilter (4) + : : : +- ^ Scan parquet (3) + : : +- ^ InputIteratorTransformer (12) + : : +- BroadcastQueryStage (10), Statistics(X) + : : +- ColumnarBroadcastExchange (9) + : : +- ^ ProjectExecTransformer (7) + : : +- ^ NoopFilter (6) + : : +- ^ Scan parquet (5) + : +- ^ FilterExecTransformer (36) + : +- ^ ProjectExecTransformer (35) + : +- ^ RegularHashAggregateExecTransformer (34) + : +- ^ InputIteratorTransformer (33) + : +- ShuffleQueryStage (31), Statistics(X) + : +- ColumnarExchange (30) + : +- ^ ProjectExecTransformer (28) + : +- ^ FlushableHashAggregateExecTransformer (27) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) + : :- ^ ProjectExecTransformer (21) + : : +- ^ NoopFilter (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (25) + : +- BroadcastQueryStage (23), Statistics(X) + : +- ReusedExchange (22) + +- ^ InputIteratorTransformer (53) + +- BroadcastQueryStage (51), Statistics(X) + +- ColumnarBroadcastExchange (50) + +- ^ ProjectExecTransformer (48) + +- ^ NoopFilter (47) + +- ^ Scan parquet (46) +- == Initial Plan == - Sort (95) - +- Exchange (94) - +- Project (93) - +- BroadcastHashJoin Inner BuildRight (92) - :- Project (87) - : +- BroadcastHashJoin LeftSemi BuildRight (86) - : :- Filter (62) - : : +- Scan parquet (61) - : +- BroadcastExchange (85) - : +- Project (84) - : +- BroadcastHashJoin Inner BuildLeft (83) - : :- BroadcastExchange (70) - : : +- BroadcastHashJoin LeftSemi BuildRight (69) - : : :- Filter (64) - : : : +- Scan parquet (63) - : : +- BroadcastExchange (68) - : : +- Project (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Filter (82) - : +- HashAggregate (81) - : +- Exchange (80) - : +- HashAggregate (79) - : +- BroadcastHashJoin LeftSemi BuildRight (78) - : :- Project (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- BroadcastExchange (77) - : +- Project (76) - : +- Filter (75) - : +- Scan parquet (74) - +- BroadcastExchange (91) - +- Project (90) - +- Filter (89) - +- Scan parquet (88) + Sort (96) + +- Exchange (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (88) + : +- BroadcastHashJoin LeftSemi BuildRight (87) + : :- Filter (63) + : : +- Scan parquet (62) + : +- BroadcastExchange (86) + : +- Project (85) + : +- BroadcastHashJoin Inner BuildLeft (84) + : :- BroadcastExchange (71) + : : +- BroadcastHashJoin LeftSemi BuildRight (70) + : : :- Filter (65) + : : : +- Scan parquet (64) + : : +- BroadcastExchange (69) + : : +- Project (68) + : : +- Filter (67) + : : +- Scan parquet (66) + : +- Filter (83) + : +- HashAggregate (82) + : +- Exchange (81) + : +- HashAggregate (80) + : +- BroadcastHashJoin LeftSemi BuildRight (79) + : :- Project (74) + : : +- Filter (73) + : : +- Scan parquet (72) + : +- BroadcastExchange (78) + : +- Project (77) + : +- Filter (76) + : +- Scan parquet (75) + +- BroadcastExchange (92) + +- Project (91) + +- Filter (90) + +- Scan parquet (89) (1) Scan parquet @@ -329,195 +330,199 @@ Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] Input [2]: [s_name#X, s_address#X] Arguments: false -(57) ColumnarExchange +(57) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(58) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(58) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(59) AQEShuffleRead +(60) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(60) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(61) Scan parquet +(62) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(63) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(63) Scan parquet +(64) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(64) Filter +(65) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(65) Scan parquet +(66) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(66) Filter +(67) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(67) Project +(68) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(69) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) BroadcastHashJoin +(70) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(70) BroadcastExchange +(71) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(71) Scan parquet +(72) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(72) Filter +(73) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(73) Project +(74) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(74) Scan parquet +(75) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(75) Filter +(76) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(76) Project +(77) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(77) BroadcastExchange +(78) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(79) HashAggregate +(80) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(80) Exchange +(81) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) HashAggregate +(82) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(82) Filter +(83) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(83) BroadcastHashJoin +(84) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(84) Project +(85) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(85) BroadcastExchange +(86) BroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(87) Project +(88) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(88) Scan parquet +(89) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(90) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(90) Project +(91) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(91) BroadcastExchange +(92) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(92) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(93) Project +(94) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(94) Exchange +(95) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Sort +(96) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(96) AdaptiveSparkPlan +(97) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt index 14b23aa966e5..89f395ed56d5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt @@ -1,84 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (91) +AdaptiveSparkPlan (92) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- BroadcastQueryStage (5), Statistics(X) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ NoopFilter (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5), Statistics(X) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ NoopFilter (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (90) - +- HashAggregate (89) - +- Exchange (88) - +- HashAggregate (87) - +- Project (86) - +- BroadcastHashJoin Inner BuildRight (85) - :- Project (80) - : +- BroadcastHashJoin Inner BuildRight (79) - : :- Project (74) - : : +- BroadcastHashJoin Inner BuildLeft (73) - : : :- BroadcastExchange (61) - : : : +- Filter (60) - : : : +- Scan parquet (59) - : : +- BroadcastHashJoin LeftAnti BuildRight (72) - : : :- BroadcastHashJoin LeftSemi BuildRight (67) - : : : :- Project (64) - : : : : +- Filter (63) - : : : : +- Scan parquet (62) - : : : +- BroadcastExchange (66) - : : : +- Scan parquet (65) - : : +- BroadcastExchange (71) - : : +- Project (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- BroadcastExchange (78) - : +- Project (77) - : +- Filter (76) - : +- Scan parquet (75) - +- BroadcastExchange (84) - +- Project (83) - +- Filter (82) - +- Scan parquet (81) + TakeOrderedAndProject (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (81) + : +- BroadcastHashJoin Inner BuildRight (80) + : :- Project (75) + : : +- BroadcastHashJoin Inner BuildLeft (74) + : : :- BroadcastExchange (62) + : : : +- Filter (61) + : : : +- Scan parquet (60) + : : +- BroadcastHashJoin LeftAnti BuildRight (73) + : : :- BroadcastHashJoin LeftSemi BuildRight (68) + : : : :- Project (65) + : : : : +- Filter (64) + : : : : +- Scan parquet (63) + : : : +- BroadcastExchange (67) + : : : +- Scan parquet (66) + : : +- BroadcastExchange (72) + : : +- Project (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- BroadcastExchange (79) + : +- Project (78) + : +- Filter (77) + : +- Scan parquet (76) + +- BroadcastExchange (85) + +- Project (84) + +- Filter (83) + +- Scan parquet (82) (1) Scan parquet @@ -290,7 +291,7 @@ Join condition: None Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(49) FlushableHashAggregateExecTransformer +(49) RegularHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] @@ -305,195 +306,199 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(53) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [2]: [s_name#X, count#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(58) VeloxColumnarToRowExec +(59) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(59) Scan parquet +(60) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(60) Filter +(61) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(61) BroadcastExchange +(62) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(62) Scan parquet +(63) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(63) Filter +(64) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(64) Project +(65) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(66) BroadcastExchange +(67) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(67) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(68) Scan parquet +(69) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(69) Filter +(70) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(70) Project +(71) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(71) BroadcastExchange +(72) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(74) Project +(75) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(75) Scan parquet +(76) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(76) Filter +(77) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(77) Project +(78) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(78) BroadcastExchange +(79) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(79) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(80) Project +(81) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(81) Scan parquet +(82) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(82) Filter +(83) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(83) Project +(84) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(84) BroadcastExchange +(85) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(85) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(86) Project +(87) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(87) HashAggregate +(88) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(88) Exchange +(89) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(90) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(90) TakeOrderedAndProject +(91) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(91) AdaptiveSparkPlan +(92) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt index 4517787bcaef..388bd109476a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (39) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (27) + +- ^ SortExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- VeloxAppendBatches (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (37) - +- Exchange (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- BroadcastHashJoin LeftAnti BuildRight (31) - :- Filter (28) - : +- Scan parquet (27) - +- BroadcastExchange (30) - +- Scan parquet (29) + Sort (38) + +- Exchange (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- BroadcastHashJoin LeftAnti BuildRight (32) + :- Filter (29) + : +- Scan parquet (28) + +- BroadcastExchange (31) + +- Scan parquet (30) (1) Scan parquet @@ -119,202 +120,206 @@ Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacc Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) ColumnarExchange +(20) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(21) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(22) InputAdapter +(23) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) SortExecTransformer +(25) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(27) Scan parquet +(28) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(28) Filter +(29) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(29) Scan parquet +(30) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(30) BroadcastExchange +(31) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(31) BroadcastHashJoin +(32) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(32) Project +(33) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(33) HashAggregate +(34) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(34) Exchange +(35) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(36) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(36) Exchange +(37) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Sort +(38) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(38) AdaptiveSparkPlan +(39) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (58) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ RegularHashAggregateExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (51) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ InputIteratorTransformer (48) + +- ShuffleQueryStage (46), Statistics(X) + +- ColumnarExchange (45) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) + HashAggregate (57) + +- Exchange (56) + +- HashAggregate (55) + +- Project (54) + +- Filter (53) + +- Scan parquet (52) -(39) Scan parquet +(40) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(40) NoopFilter +(41) NoopFilter Input [2]: [c_phone#X, c_acctbal#X] Arguments: [c_phone#X, c_acctbal#X] -(41) ProjectExecTransformer +(42) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(42) FlushableHashAggregateExecTransformer +(43) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(43) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(44) ColumnarExchange +(45) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(46) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(46) InputAdapter +(47) InputAdapter Input [2]: [sum#X, count#X] -(47) InputIteratorTransformer +(48) InputIteratorTransformer Input [2]: [sum#X, count#X] -(48) RegularHashAggregateExecTransformer +(49) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(49) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(50) VeloxColumnarToRowExec +(51) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(51) Scan parquet +(52) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(52) Filter +(53) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(53) Project +(54) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(54) HashAggregate +(55) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(55) Exchange +(56) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(57) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) AdaptiveSparkPlan +(58) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt index 361ea070d42a..c157058fc2ef 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- TakeOrderedAndProjectExecTransformer (33) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- BroadcastQueryStage (18), Statistics(X) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ NoopFilter (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (35) + +- TakeOrderedAndProjectExecTransformer (34) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ RegularHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ NoopFilter (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildLeft (41) - : :- BroadcastExchange (38) - : : +- Project (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + TakeOrderedAndProject (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- BroadcastHashJoin Inner BuildRight (48) + :- Project (43) + : +- BroadcastHashJoin Inner BuildLeft (42) + : :- BroadcastExchange (39) + : : +- Project (38) + : : +- Filter (37) + : : +- Scan parquet (36) + : +- Filter (41) + : +- Scan parquet (40) + +- BroadcastExchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -146,7 +147,7 @@ Join condition: None Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) FlushableHashAggregateExecTransformer +(23) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -161,133 +162,137 @@ Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: X + +(27) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(33) TakeOrderedAndProjectExecTransformer +(34) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(34) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(35) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(36) Filter +(37) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(37) Project +(38) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(38) BroadcastExchange +(39) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) Scan parquet +(40) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(42) Project +(43) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(43) Scan parquet +(44) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(45) Project +(46) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) BroadcastExchange +(47) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(48) Project +(49) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(49) HashAggregate +(50) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(50) Exchange +(51) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(52) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(52) TakeOrderedAndProject +(53) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt index 4bdcd640058c..5e3f77bb06da 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt @@ -1,43 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9), Statistics(X) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- BroadcastHashJoin LeftSemi BuildRight (37) - :- Project (32) - : +- Filter (31) - : +- Scan parquet (30) - +- BroadcastExchange (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (44) + +- Exchange (43) + +- HashAggregate (42) + +- Exchange (41) + +- HashAggregate (40) + +- Project (39) + +- BroadcastHashJoin LeftSemi BuildRight (38) + :- Project (33) + : +- Filter (32) + : +- Scan parquet (31) + +- BroadcastExchange (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -138,101 +139,105 @@ Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(27) SortExecTransformer +(28) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(30) Scan parquet +(31) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(31) Filter +(32) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(32) Project +(33) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(33) Scan parquet +(34) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(34) Filter +(35) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(35) Project +(36) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(36) BroadcastExchange +(37) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(37) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(38) Project +(39) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(39) HashAggregate +(40) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(40) Exchange +(41) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) HashAggregate +(42) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(42) Exchange +(43) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(44) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt index d1f0cb5eba56..0f5d19f82450 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt @@ -1,91 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (100) +AdaptiveSparkPlan (101) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45), Statistics(X) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61), Statistics(X) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (99) - +- Exchange (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Project (71) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (92) - +- Project (91) - +- Filter (90) - +- Scan parquet (89) + Sort (100) + +- Exchange (99) + +- HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- BroadcastHashJoin Inner BuildRight (94) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Project (72) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (93) + +- Project (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -342,201 +343,205 @@ Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS reven Input [2]: [n_name#X, revenue#X] Arguments: false -(59) ColumnarExchange +(59) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(60) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(61) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(61) InputAdapter +(62) InputAdapter Input [2]: [n_name#X, revenue#X] -(62) InputIteratorTransformer +(63) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(63) SortExecTransformer +(64) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(65) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(67) Filter +(68) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(68) BroadcastExchange +(69) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(71) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(73) Project +(74) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(75) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(78) Project +(79) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(79) Scan parquet +(80) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(81) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(81) BroadcastExchange +(82) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(83) Project +(84) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(84) Scan parquet +(85) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(86) BroadcastExchange +(87) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(88) Project +(89) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(89) Scan parquet +(90) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(91) Project +(92) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(92) BroadcastExchange +(93) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(93) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(94) Project +(95) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(95) HashAggregate +(96) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(96) Exchange +(97) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(98) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(98) Exchange +(99) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Sort +(100) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(100) AdaptiveSparkPlan +(101) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt index 5987a808f5fd..fbcc75a247d5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * l_discount#X) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt index 29c2524a7615..7a9949d5358c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt @@ -1,85 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ SortExecTransformer (58) - +- ^ InputIteratorTransformer (57) - +- ShuffleQueryStage (55), Statistics(X) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16), Statistics(X) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ NoopFilter (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25), Statistics(X) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ NoopFilter (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ NoopFilter (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40), Statistics(X) - +- ReusedExchange (39) + VeloxColumnarToRowExec (61) + +- ^ SortExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- VeloxAppendBatches (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ NoopFilter (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ NoopFilter (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == - Sort (92) - +- Exchange (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (63) - : : : : : +- Filter (62) - : : : : : +- Scan parquet (61) - : : : : +- Filter (65) - : : : : +- Scan parquet (64) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (85) - +- Filter (84) - +- Scan parquet (83) + Sort (93) + +- Exchange (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- BroadcastHashJoin Inner BuildRight (87) + :- Project (83) + : +- BroadcastHashJoin Inner BuildRight (82) + : :- Project (78) + : : +- BroadcastHashJoin Inner BuildRight (77) + : : :- Project (73) + : : : +- BroadcastHashJoin Inner BuildRight (72) + : : : :- Project (68) + : : : : +- BroadcastHashJoin Inner BuildLeft (67) + : : : : :- BroadcastExchange (64) + : : : : : +- Filter (63) + : : : : : +- Scan parquet (62) + : : : : +- Filter (66) + : : : : +- Scan parquet (65) + : : : +- BroadcastExchange (71) + : : : +- Filter (70) + : : : +- Scan parquet (69) + : : +- BroadcastExchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- BroadcastExchange (81) + : +- Filter (80) + : +- Scan parquet (79) + +- BroadcastExchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -312,193 +313,197 @@ Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) ColumnarExchange +(54) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(55) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(56) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(56) InputAdapter +(57) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(57) InputIteratorTransformer +(58) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) SortExecTransformer +(59) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(59) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(60) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(61) Scan parquet +(62) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(63) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(63) BroadcastExchange +(64) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(64) Scan parquet +(65) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(65) Filter +(66) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(66) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(67) Project +(68) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(68) Scan parquet +(69) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(69) Filter +(70) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(70) BroadcastExchange +(71) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(72) Project +(73) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(77) Project +(78) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(79) Filter +(80) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(80) BroadcastExchange +(81) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(82) Project +(83) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(87) Project +(88) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(88) HashAggregate +(89) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(89) Exchange +(90) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(91) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(91) Exchange +(92) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(93) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(93) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt index 715bc6f4e5f8..e5fd2360e758 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt @@ -1,116 +1,117 @@ == Physical Plan == -AdaptiveSparkPlan (129) +AdaptiveSparkPlan (130) +- == Final Plan == - VeloxColumnarToRowExec (84) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ NoopFilter (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26), Statistics(X) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ NoopFilter (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35), Statistics(X) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ NoopFilter (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44), Statistics(X) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ NoopFilter (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53), Statistics(X) - : +- ColumnarBroadcastExchange (52) - : +- ^ NoopFilter (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63), Statistics(X) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ NoopFilter (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (85) + +- ^ SortExecTransformer (83) + +- ^ InputIteratorTransformer (82) + +- ShuffleQueryStage (80), Statistics(X) + +- ColumnarExchange (79) + +- VeloxAppendBatches (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ShuffleQueryStage (72), Statistics(X) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ NoopFilter (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ NoopFilter (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ NoopFilter (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ NoopFilter (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ NoopFilter (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ NoopFilter (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (128) - +- Exchange (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- BroadcastHashJoin Inner BuildRight (122) - :- Project (117) - : +- BroadcastHashJoin Inner BuildRight (116) - : :- Project (112) - : : +- BroadcastHashJoin Inner BuildRight (111) - : : :- Project (107) - : : : +- BroadcastHashJoin Inner BuildRight (106) - : : : :- Project (102) - : : : : +- BroadcastHashJoin Inner BuildRight (101) - : : : : :- Project (97) - : : : : : +- BroadcastHashJoin Inner BuildRight (96) - : : : : : :- Project (92) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) - : : : : : : :- BroadcastExchange (88) - : : : : : : : +- Project (87) - : : : : : : : +- Filter (86) - : : : : : : : +- Scan parquet (85) - : : : : : : +- Filter (90) - : : : : : : +- Scan parquet (89) - : : : : : +- BroadcastExchange (95) - : : : : : +- Filter (94) - : : : : : +- Scan parquet (93) - : : : : +- BroadcastExchange (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- BroadcastExchange (105) - : : : +- Filter (104) - : : : +- Scan parquet (103) - : : +- BroadcastExchange (110) - : : +- Filter (109) - : : +- Scan parquet (108) - : +- BroadcastExchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- BroadcastExchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (129) + +- Exchange (128) + +- HashAggregate (127) + +- Exchange (126) + +- HashAggregate (125) + +- Project (124) + +- BroadcastHashJoin Inner BuildRight (123) + :- Project (118) + : +- BroadcastHashJoin Inner BuildRight (117) + : :- Project (113) + : : +- BroadcastHashJoin Inner BuildRight (112) + : : :- Project (108) + : : : +- BroadcastHashJoin Inner BuildRight (107) + : : : :- Project (103) + : : : : +- BroadcastHashJoin Inner BuildRight (102) + : : : : :- Project (98) + : : : : : +- BroadcastHashJoin Inner BuildRight (97) + : : : : : :- Project (93) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (92) + : : : : : : :- BroadcastExchange (89) + : : : : : : : +- Project (88) + : : : : : : : +- Filter (87) + : : : : : : : +- Scan parquet (86) + : : : : : : +- Filter (91) + : : : : : : +- Scan parquet (90) + : : : : : +- BroadcastExchange (96) + : : : : : +- Filter (95) + : : : : : +- Scan parquet (94) + : : : : +- BroadcastExchange (101) + : : : : +- Filter (100) + : : : : +- Scan parquet (99) + : : : +- BroadcastExchange (106) + : : : +- Filter (105) + : : : +- Scan parquet (104) + : : +- BroadcastExchange (111) + : : +- Filter (110) + : : +- Scan parquet (109) + : +- BroadcastExchange (116) + : +- Filter (115) + : +- Scan parquet (114) + +- BroadcastExchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) (1) Scan parquet @@ -449,251 +450,255 @@ Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.000 Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) ColumnarExchange +(78) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(79) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(80) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(80) InputAdapter +(81) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(81) InputIteratorTransformer +(82) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(82) SortExecTransformer +(83) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(83) WholeStageCodegenTransformer (X) +(84) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(84) VeloxColumnarToRowExec +(85) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(85) Scan parquet +(86) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(87) Project +(88) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(88) BroadcastExchange +(89) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) Scan parquet +(90) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(92) Project +(93) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(93) Scan parquet +(94) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(94) Filter +(95) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(95) BroadcastExchange +(96) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(96) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(97) Project +(98) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(98) Scan parquet +(99) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(99) Filter +(100) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(100) BroadcastExchange +(101) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(101) BroadcastHashJoin +(102) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(102) Project +(103) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(103) Scan parquet +(104) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(104) Filter +(105) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(105) BroadcastExchange +(106) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(106) BroadcastHashJoin +(107) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(107) Project +(108) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(108) Scan parquet +(109) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(109) Filter +(110) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(110) BroadcastExchange +(111) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(111) BroadcastHashJoin +(112) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(112) Project +(113) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(113) Scan parquet +(114) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(114) Filter +(115) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(115) BroadcastExchange +(116) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(116) BroadcastHashJoin +(117) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(117) Project +(118) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(118) Scan parquet +(119) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(119) Filter +(120) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(120) Project +(121) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(121) BroadcastExchange +(122) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(122) BroadcastHashJoin +(123) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(123) Project +(124) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(124) HashAggregate +(125) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) Exchange +(126) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(127) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(127) Exchange +(128) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(129) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(129) AdaptiveSparkPlan +(130) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt index 14c32b04eb48..4538df9c485a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt @@ -1,89 +1,90 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (99) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- ^ SortExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (65) + +- ^ SortExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (97) - +- Exchange (96) - +- HashAggregate (95) - +- Exchange (94) - +- HashAggregate (93) - +- Project (92) - +- BroadcastHashJoin Inner BuildRight (91) - :- Project (87) - : +- BroadcastHashJoin Inner BuildRight (86) - : :- Project (82) - : : +- BroadcastHashJoin Inner BuildRight (81) - : : :- Project (77) - : : : +- BroadcastHashJoin Inner BuildRight (76) - : : : :- Project (72) - : : : : +- BroadcastHashJoin Inner BuildLeft (71) - : : : : :- BroadcastExchange (68) - : : : : : +- Project (67) - : : : : : +- Filter (66) - : : : : : +- Scan parquet (65) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (75) - : : : +- Filter (74) - : : : +- Scan parquet (73) - : : +- BroadcastExchange (80) - : : +- Filter (79) - : : +- Scan parquet (78) - : +- BroadcastExchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- BroadcastExchange (90) - +- Filter (89) - +- Scan parquet (88) + Sort (98) + +- Exchange (97) + +- HashAggregate (96) + +- Exchange (95) + +- HashAggregate (94) + +- Project (93) + +- BroadcastHashJoin Inner BuildRight (92) + :- Project (88) + : +- BroadcastHashJoin Inner BuildRight (87) + : :- Project (83) + : : +- BroadcastHashJoin Inner BuildRight (82) + : : :- Project (78) + : : : +- BroadcastHashJoin Inner BuildRight (77) + : : : :- Project (73) + : : : : +- BroadcastHashJoin Inner BuildLeft (72) + : : : : :- BroadcastExchange (69) + : : : : : +- Project (68) + : : : : : +- Filter (67) + : : : : : +- Scan parquet (66) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (76) + : : : +- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- BroadcastExchange (86) + : +- Filter (85) + : +- Scan parquet (84) + +- BroadcastExchange (91) + +- Filter (90) + +- Scan parquet (89) (1) Scan parquet @@ -336,197 +337,201 @@ Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) ColumnarExchange +(58) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(59) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(60) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(60) InputAdapter +(61) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(61) InputIteratorTransformer +(62) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) SortExecTransformer +(63) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(63) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(64) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(67) Project +(68) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(69) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(72) Project +(73) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(77) Project +(78) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(79) Filter +(80) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(80) BroadcastExchange +(81) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(82) Project +(83) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(87) Project +(88) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(88) Scan parquet +(89) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(90) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(90) BroadcastExchange +(91) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(92) Project +(93) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(93) HashAggregate +(94) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(94) Exchange +(95) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) HashAggregate +(96) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(96) Exchange +(97) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(98) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(98) AdaptiveSparkPlan +(99) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/1.txt index 656f98574483..1c8670c0fea4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt index 3afadbc4c678..61a6de21a400 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt @@ -1,64 +1,65 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ShuffleQueryStage (36) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- BroadcastQueryStage (8) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ FilterExecTransformer (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- BroadcastQueryStage (18) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ FilterExecTransformer (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- BroadcastQueryStage (27) - +- ColumnarBroadcastExchange (26) - +- ^ FilterExecTransformer (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (44) + +- TakeOrderedAndProjectExecTransformer (43) + +- ^ ProjectExecTransformer (41) + +- ^ RegularHashAggregateExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ShuffleQueryStage (37) + +- ColumnarExchange (36) + +- VeloxAppendBatches (35) + +- ^ ProjectExecTransformer (33) + +- ^ RegularHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ FilterExecTransformer (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ FilterExecTransformer (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27) + +- ColumnarBroadcastExchange (26) + +- ^ FilterExecTransformer (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- BroadcastHashJoin Inner BuildRight (61) - :- Project (57) - : +- BroadcastHashJoin Inner BuildRight (56) - : :- Project (51) - : : +- BroadcastHashJoin Inner BuildRight (50) - : : :- Filter (45) - : : : +- Scan parquet (44) - : : +- BroadcastExchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (55) - : +- Project (54) - : +- Filter (53) - : +- Scan parquet (52) - +- BroadcastExchange (60) - +- Filter (59) - +- Scan parquet (58) + TakeOrderedAndProject (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- BroadcastHashJoin Inner BuildRight (62) + :- Project (58) + : +- BroadcastHashJoin Inner BuildRight (57) + : :- Project (52) + : : +- BroadcastHashJoin Inner BuildRight (51) + : : :- Filter (46) + : : : +- Scan parquet (45) + : : +- BroadcastExchange (50) + : : +- Project (49) + : : +- Filter (48) + : : +- Scan parquet (47) + : +- BroadcastExchange (56) + : +- Project (55) + : +- Filter (54) + : +- Scan parquet (53) + +- BroadcastExchange (61) + +- Filter (60) + +- Scan parquet (59) (1) Scan parquet @@ -194,7 +195,7 @@ Join condition: None Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(32) FlushableHashAggregateExecTransformer +(32) RegularHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -209,155 +210,159 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(35) ColumnarExchange +(35) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(36) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(37) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(37) InputAdapter +(38) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(38) InputIteratorTransformer +(39) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(40) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(40) ProjectExecTransformer +(41) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(41) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(43) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(43) VeloxColumnarToRowExec +(44) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(44) Scan parquet +(45) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(46) Scan parquet +(47) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(48) Project +(49) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(51) Project +(52) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(52) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(53) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(54) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(55) BroadcastExchange +(56) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(57) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(58) Scan parquet +(59) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(59) Filter +(60) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(60) BroadcastExchange +(61) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(61) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) Project +(63) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(63) HashAggregate +(64) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(64) Exchange +(65) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(66) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(66) TakeOrderedAndProject +(67) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt index 03199cd9feb1..72b0287a9dbe 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt @@ -1,55 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7) - : +- ColumnarBroadcastExchange (6) - : +- ^ FilterExecTransformer (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FilterExecTransformer (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7) + : +- ColumnarBroadcastExchange (6) + : +- ^ FilterExecTransformer (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FilterExecTransformer (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (57) - +- Exchange (56) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- BroadcastHashJoin Inner BuildRight (50) - :- Project (45) - : +- BroadcastHashJoin Inner BuildRight (44) - : :- Filter (40) - : : +- Scan parquet (39) - : +- BroadcastExchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- BroadcastExchange (49) - +- Project (48) - +- Filter (47) - +- Scan parquet (46) + Sort (58) + +- Exchange (57) + +- Filter (56) + +- HashAggregate (55) + +- Exchange (54) + +- HashAggregate (53) + +- Project (52) + +- BroadcastHashJoin Inner BuildRight (51) + :- Project (46) + : +- BroadcastHashJoin Inner BuildRight (45) + : :- Filter (41) + : : +- Scan parquet (40) + : +- BroadcastExchange (44) + : +- Filter (43) + : +- Scan parquet (42) + +- BroadcastExchange (50) + +- Project (49) + +- Filter (48) + +- Scan parquet (47) (1) Scan parquet @@ -187,124 +188,128 @@ Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery su Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(34) InputAdapter +(35) InputAdapter Input [2]: [ps_partkey#X, value#X] -(35) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(36) SortExecTransformer +(37) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(38) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(39) Scan parquet +(40) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(41) Scan parquet +(42) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(42) Filter +(43) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(43) BroadcastExchange +(44) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(45) Project +(46) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(46) Scan parquet +(47) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(48) Project +(49) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(51) Project +(52) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(52) HashAggregate +(53) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(53) Exchange +(54) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(55) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(55) Filter +(56) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(56) Exchange +(57) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(58) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(58) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt index e78891a522ff..23ab061662fc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt @@ -1,41 +1,42 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (43) +- == Final Plan == - VeloxColumnarToRowExec (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (29) + +- ^ SortExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- VeloxAppendBatches (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) + Sort (42) + +- Exchange (41) + +- HashAggregate (40) + +- Exchange (39) + +- HashAggregate (38) + +- Project (37) + +- BroadcastHashJoin Inner BuildLeft (36) + :- BroadcastExchange (32) + : +- Filter (31) + : +- Scan parquet (30) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -131,96 +132,100 @@ Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_ Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) SortExecTransformer +(27) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(27) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(30) Filter +(31) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(31) BroadcastExchange +(32) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(32) Scan parquet +(33) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(33) Filter +(34) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(34) Project +(35) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(35) BroadcastHashJoin +(36) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) Project +(37) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(37) HashAggregate +(38) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(38) Exchange +(39) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(40) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(40) Exchange +(41) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(42) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(43) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt index 9aa658b2d78d..0f55c5334af5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt @@ -1,49 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FilterExecTransformer (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ FilterExecTransformer (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- BroadcastHashJoin LeftOuter BuildRight (42) - :- Scan parquet (37) - +- BroadcastExchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- BroadcastHashJoin LeftOuter BuildRight (43) + :- Scan parquet (38) + +- BroadcastExchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -174,109 +175,113 @@ Results [2]: [c_count#X, count(1)#X AS custdist#X] Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(33) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(35) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(38) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(44) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) Exchange +(46) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(48) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(49) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(50) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(51) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(52) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt index fcd550bdb92c..2427b05691d0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt @@ -1,35 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- BroadcastHashJoin Inner BuildRight (29) - :- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- BroadcastExchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- BroadcastHashJoin Inner BuildRight (30) + :- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- BroadcastExchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -85,7 +86,7 @@ Join condition: None Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] @@ -96,95 +97,99 @@ Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(23) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(25) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) Scan parquet +(27) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(28) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) BroadcastExchange +(29) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastHashJoin +(30) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(30) Project +(31) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(32) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) Exchange +(33) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] -(34) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt index 54dad76174f9..64f59fa9c811 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt @@ -1,43 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- Project (41) - +- BroadcastHashJoin Inner BuildLeft (40) - :- BroadcastExchange (32) - : +- Filter (31) - : +- Scan parquet (30) - +- Filter (39) - +- HashAggregate (38) - +- Exchange (37) - +- HashAggregate (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (44) + +- Exchange (43) + +- Project (42) + +- BroadcastHashJoin Inner BuildLeft (41) + :- BroadcastExchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Filter (40) + +- HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -137,100 +138,104 @@ Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_ Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) SortExecTransformer +(28) SortExecTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(30) Scan parquet +(31) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(32) BroadcastExchange +(33) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(33) Scan parquet +(34) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(34) Filter +(35) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(35) Project +(36) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(36) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(37) Exchange +(38) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) HashAggregate +(39) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(39) Filter +(40) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(40) BroadcastHashJoin +(41) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(41) Project +(42) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(42) Exchange +(43) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(44) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt index 5197f57218a4..7411f2a6667a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt @@ -1,53 +1,54 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (57) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ShuffleQueryStage (30) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ FilterExecTransformer (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ FilterExecTransformer (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- BroadcastHashJoin Inner BuildRight (46) - :- BroadcastHashJoin LeftAnti BuildRight (42) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (45) - +- Filter (44) - +- Scan parquet (43) + Sort (56) + +- Exchange (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- BroadcastHashJoin Inner BuildRight (47) + :- BroadcastHashJoin LeftAnti BuildRight (43) + : :- Filter (38) + : : +- Scan parquet (37) + : +- BroadcastExchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- BroadcastExchange (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -175,134 +176,138 @@ Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_c Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) ColumnarExchange +(29) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(30) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(36) Scan parquet +(37) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(37) Filter +(38) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(38) Scan parquet +(39) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(39) Filter +(40) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(40) Project +(41) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(43) Scan parquet +(44) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(45) BroadcastExchange +(46) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(47) Project +(48) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(48) HashAggregate +(49) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(49) Exchange +(50) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(51) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(51) HashAggregate +(52) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(52) Exchange +(53) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(54) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(56) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(56) AdaptiveSparkPlan +(57) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/17.txt index 9d81e1d5053f..1b258da06f20 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/17.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (14) - +- ^ ProjectExecTransformer (12) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- RowToVeloxColumnar (2) - +- LocalTableScan (1) + VeloxColumnarToRowExec (15) + +- ^ ProjectExecTransformer (13) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ RegularHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == - HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin Inner BuildRight (30) - :- Project (22) - : +- BroadcastHashJoin Inner BuildRight (21) - : :- Filter (16) - : : +- Scan parquet (15) - : +- BroadcastExchange (20) - : +- Project (19) - : +- Filter (18) - : +- Scan parquet (17) - +- BroadcastExchange (29) - +- Filter (28) - +- HashAggregate (27) - +- Exchange (26) - +- HashAggregate (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin Inner BuildRight (31) + :- Project (23) + : +- BroadcastHashJoin Inner BuildRight (22) + : :- Filter (17) + : : +- Scan parquet (16) + : +- BroadcastExchange (21) + : +- Project (20) + : +- Filter (19) + : +- Scan parquet (18) + +- BroadcastExchange (30) + +- Filter (29) + +- HashAggregate (28) + +- Exchange (27) + +- HashAggregate (26) + +- Filter (25) + +- Scan parquet (24) (1) LocalTableScan @@ -47,7 +48,7 @@ Input [1]: [l_extendedprice#X] (4) InputIteratorTransformer Input [1]: [l_extendedprice#X] -(5) FlushableHashAggregateExecTransformer +(5) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] @@ -58,141 +59,145 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(8) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [2]: [sum#X, isEmpty#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(14) VeloxColumnarToRowExec +(15) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(15) Scan parquet +(16) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(16) Filter +(17) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(17) Scan parquet +(18) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(18) Filter +(19) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(19) Project +(20) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(20) BroadcastExchange +(21) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastHashJoin +(22) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(22) Project +(23) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(23) Scan parquet +(24) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(25) HashAggregate +(26) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(26) Exchange +(27) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) HashAggregate +(28) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(28) Filter +(29) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(31) Project +(32) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(32) HashAggregate +(33) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] -(35) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt index 341589565740..ced5a3b68cc4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt @@ -1,80 +1,82 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- TakeOrderedAndProjectExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ShuffleQueryStage (47) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- BroadcastQueryStage (5) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ FilterExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- BroadcastQueryStage (38) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- BroadcastQueryStage (32) - +- ReusedExchange (31) + VeloxColumnarToRowExec (55) + +- TakeOrderedAndProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49) + +- ColumnarExchange (48) + +- VeloxAppendBatches (47) + +- ^ ProjectExecTransformer (45) + +- ^ RegularHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) + :- ^ ProjectExecTransformer (29) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (28) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (27) + : :- ^ FilterExecTransformer (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (26) + : +- BroadcastQueryStage (24) + : +- ColumnarBroadcastExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FilterExecTransformer (20) + : +- ^ RegularHashAggregateExecTransformer (19) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ RegularHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (41) + +- BroadcastQueryStage (39) + +- ColumnarBroadcastExchange (38) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (36) + :- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (35) + +- BroadcastQueryStage (33) + +- ReusedExchange (32) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (68) - : +- BroadcastHashJoin Inner BuildLeft (67) - : :- BroadcastExchange (56) - : : +- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastHashJoin LeftSemi BuildRight (66) - : :- Filter (58) - : : +- Scan parquet (57) - : +- BroadcastExchange (65) - : +- Project (64) - : +- Filter (63) - : +- HashAggregate (62) - : +- Exchange (61) - : +- HashAggregate (60) - : +- Scan parquet (59) - +- BroadcastExchange (79) - +- BroadcastHashJoin LeftSemi BuildRight (78) - :- Filter (70) - : +- Scan parquet (69) - +- BroadcastExchange (77) - +- Project (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Scan parquet (71) + TakeOrderedAndProject (87) + +- HashAggregate (86) + +- Exchange (85) + +- HashAggregate (84) + +- Project (83) + +- BroadcastHashJoin Inner BuildRight (82) + :- Project (70) + : +- BroadcastHashJoin Inner BuildLeft (69) + : :- BroadcastExchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- BroadcastHashJoin LeftSemi BuildRight (68) + : :- Filter (60) + : : +- Scan parquet (59) + : +- BroadcastExchange (67) + : +- Project (66) + : +- Filter (65) + : +- HashAggregate (64) + : +- Exchange (63) + : +- HashAggregate (62) + : +- Scan parquet (61) + +- BroadcastExchange (81) + +- BroadcastHashJoin LeftSemi BuildRight (80) + :- Filter (72) + : +- Scan parquet (71) + +- BroadcastExchange (79) + +- Project (78) + +- Filter (77) + +- HashAggregate (76) + +- Exchange (75) + +- HashAggregate (74) + +- Scan parquet (73) (1) Scan parquet @@ -123,7 +125,7 @@ Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) FlushableHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] @@ -138,333 +140,341 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [1]: [l_orderkey#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [1]: [l_orderkey#X] -(26) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(27) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(30) FilterExecTransformer +(31) FilterExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Arguments: isnotnull(l_orderkey#X) -(31) ReusedExchange [Reuses operator id: 22] +(32) ReusedExchange [Reuses operator id: 23] Output [1]: [l_orderkey#X] -(32) BroadcastQueryStage +(33) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [1]: [l_orderkey#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [l_orderkey#X] -(35) BroadcastHashJoinExecTransformer +(36) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(37) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(39) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(41) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(43) FlushableHashAggregateExecTransformer +(44) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(47) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: X + +(48) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(49) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(50) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(51) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(51) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(52) TakeOrderedAndProjectExecTransformer +(54) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(53) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(54) Scan parquet +(56) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(56) BroadcastExchange +(58) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(59) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(58) Filter +(60) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(59) Scan parquet +(61) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(60) HashAggregate +(62) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(61) Exchange +(63) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(63) Filter +(65) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(64) Project +(66) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(65) BroadcastExchange +(67) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(67) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(68) Project +(70) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(69) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(71) Scan parquet +(73) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(74) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(75) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(76) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(77) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(78) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) BroadcastExchange +(79) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(79) BroadcastExchange +(81) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(81) Project +(83) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(82) HashAggregate +(84) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(83) Exchange +(85) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(86) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(85) TakeOrderedAndProject +(87) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(86) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt index a46daacbdb75..02f21577ad17 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt @@ -1,34 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (32) - +- Exchange (31) - +- HashAggregate (30) - +- Project (29) - +- BroadcastHashJoin Inner BuildRight (28) - :- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- BroadcastExchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -84,7 +85,7 @@ Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -95,91 +96,95 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(20) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(22) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(23) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(24) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(25) Scan parquet +(26) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(27) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) BroadcastExchange +(28) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(30) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(31) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) Exchange +(32) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(32) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(33) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt index 6fcedbbda996..c5b3f4cc9283 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt @@ -1,97 +1,99 @@ == Physical Plan == -AdaptiveSparkPlan (107) +AdaptiveSparkPlan (109) +- == Final Plan == - VeloxColumnarToRowExec (70) - +- ^ SortExecTransformer (68) - +- ^ InputIteratorTransformer (67) - +- ShuffleQueryStage (65) - +- ColumnarExchange (64) - +- ^ ProjectExecTransformer (62) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (61) - :- ^ ProjectExecTransformer (52) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (51) - : :- ^ InputIteratorTransformer (9) - : : +- AQEShuffleRead (7) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (50) - : +- BroadcastQueryStage (48) - : +- ColumnarBroadcastExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) - : :- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) - : : :- ^ FilterExecTransformer (11) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (19) - : : +- BroadcastQueryStage (17) - : : +- ColumnarBroadcastExchange (16) - : : +- ^ ProjectExecTransformer (14) - : : +- ^ FilterExecTransformer (13) - : : +- ^ Scan parquet (12) - : +- ^ FilterExecTransformer (43) - : +- ^ ProjectExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ InputIteratorTransformer (40) - : +- ShuffleQueryStage (38) - : +- ColumnarExchange (37) - : +- ^ ProjectExecTransformer (35) - : +- ^ FlushableHashAggregateExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) - : :- ^ ProjectExecTransformer (28) - : : +- ^ FilterExecTransformer (27) - : : +- ^ Scan parquet (26) - : +- ^ InputIteratorTransformer (32) - : +- BroadcastQueryStage (30) - : +- ReusedExchange (29) - +- ^ InputIteratorTransformer (60) - +- BroadcastQueryStage (58) - +- ColumnarBroadcastExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ FilterExecTransformer (54) - +- ^ Scan parquet (53) + VeloxColumnarToRowExec (72) + +- ^ SortExecTransformer (70) + +- ^ InputIteratorTransformer (69) + +- ShuffleQueryStage (67) + +- ColumnarExchange (66) + +- VeloxAppendBatches (65) + +- ^ ProjectExecTransformer (63) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (62) + :- ^ ProjectExecTransformer (53) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (52) + : :- ^ InputIteratorTransformer (10) + : : +- AQEShuffleRead (8) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (51) + : +- BroadcastQueryStage (49) + : +- ColumnarBroadcastExchange (48) + : +- ^ ProjectExecTransformer (46) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (45) + : :- ^ InputIteratorTransformer (26) + : : +- BroadcastQueryStage (24) + : : +- ColumnarBroadcastExchange (23) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (21) + : : :- ^ FilterExecTransformer (12) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (20) + : : +- BroadcastQueryStage (18) + : : +- ColumnarBroadcastExchange (17) + : : +- ^ ProjectExecTransformer (15) + : : +- ^ FilterExecTransformer (14) + : : +- ^ Scan parquet (13) + : +- ^ FilterExecTransformer (44) + : +- ^ ProjectExecTransformer (43) + : +- ^ RegularHashAggregateExecTransformer (42) + : +- ^ InputIteratorTransformer (41) + : +- ShuffleQueryStage (39) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ FlushableHashAggregateExecTransformer (35) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (34) + : :- ^ ProjectExecTransformer (29) + : : +- ^ FilterExecTransformer (28) + : : +- ^ Scan parquet (27) + : +- ^ InputIteratorTransformer (33) + : +- BroadcastQueryStage (31) + : +- ReusedExchange (30) + +- ^ InputIteratorTransformer (61) + +- BroadcastQueryStage (59) + +- ColumnarBroadcastExchange (58) + +- ^ ProjectExecTransformer (56) + +- ^ FilterExecTransformer (55) + +- ^ Scan parquet (54) +- == Initial Plan == - Sort (106) - +- Exchange (105) - +- Project (104) - +- BroadcastHashJoin Inner BuildRight (103) - :- Project (98) - : +- ShuffledHashJoin LeftSemi BuildRight (97) - : :- Exchange (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (96) - : +- Project (95) - : +- BroadcastHashJoin Inner BuildLeft (94) - : :- BroadcastExchange (81) - : : +- BroadcastHashJoin LeftSemi BuildRight (80) - : : :- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (79) - : : +- Project (78) - : : +- Filter (77) - : : +- Scan parquet (76) - : +- Filter (93) - : +- HashAggregate (92) - : +- Exchange (91) - : +- HashAggregate (90) - : +- BroadcastHashJoin LeftSemi BuildRight (89) - : :- Project (84) - : : +- Filter (83) - : : +- Scan parquet (82) - : +- BroadcastExchange (88) - : +- Project (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (102) - +- Project (101) - +- Filter (100) - +- Scan parquet (99) + Sort (108) + +- Exchange (107) + +- Project (106) + +- BroadcastHashJoin Inner BuildRight (105) + :- Project (100) + : +- ShuffledHashJoin LeftSemi BuildRight (99) + : :- Exchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- Exchange (98) + : +- Project (97) + : +- BroadcastHashJoin Inner BuildLeft (96) + : :- BroadcastExchange (83) + : : +- BroadcastHashJoin LeftSemi BuildRight (82) + : : :- Filter (77) + : : : +- Scan parquet (76) + : : +- BroadcastExchange (81) + : : +- Project (80) + : : +- Filter (79) + : : +- Scan parquet (78) + : +- Filter (95) + : +- HashAggregate (94) + : +- Exchange (93) + : +- HashAggregate (92) + : +- BroadcastHashJoin LeftSemi BuildRight (91) + : :- Project (86) + : : +- Filter (85) + : : +- Scan parquet (84) + : +- BroadcastExchange (90) + : +- Project (89) + : +- Filter (88) + : +- Scan parquet (87) + +- BroadcastExchange (104) + +- Project (103) + +- Filter (102) + +- Scan parquet (101) (1) Scan parquet @@ -113,448 +115,456 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) AQEShuffleRead +(8) AQEShuffleRead Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: local -(8) InputAdapter +(9) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(10) Scan parquet +(11) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(11) FilterExecTransformer +(12) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(12) Scan parquet +(13) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(13) FilterExecTransformer +(14) FilterExecTransformer Input [2]: [p_partkey#X, p_name#X] Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(14) ProjectExecTransformer +(15) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(15) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(16) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(17) BroadcastQueryStage +(18) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [1]: [p_partkey#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [1]: [p_partkey#X] -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(26) Scan parquet +(27) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(27) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(29) ReusedExchange [Reuses operator id: 16] +(30) ReusedExchange [Reuses operator id: 17] Output [1]: [p_partkey#X] -(30) BroadcastQueryStage +(31) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [1]: [p_partkey#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [1]: [p_partkey#X] -(33) BroadcastHashJoinExecTransformer +(34) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(34) FlushableHashAggregateExecTransformer +(35) FlushableHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(36) ProjectExecTransformer Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(38) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(39) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(42) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) FilterExecTransformer +(44) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(44) BroadcastHashJoinExecTransformer +(45) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(45) ProjectExecTransformer +(46) ProjectExecTransformer Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(46) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] Arguments: false -(47) ColumnarBroadcastExchange +(48) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(48) BroadcastQueryStage +(49) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(49) InputAdapter +(50) InputAdapter Input [1]: [ps_suppkey#X] -(50) InputIteratorTransformer +(51) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(51) BroadcastHashJoinExecTransformer +(52) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(52) ProjectExecTransformer +(53) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(53) Scan parquet +(54) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(54) FilterExecTransformer +(55) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(55) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(56) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(57) ColumnarBroadcastExchange +(58) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(58) BroadcastQueryStage +(59) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(59) InputAdapter +(60) InputAdapter Input [1]: [n_nationkey#X] -(60) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [n_nationkey#X] -(61) BroadcastHashJoinExecTransformer +(62) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) ProjectExecTransformer +(63) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(63) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(64) ColumnarExchange +(65) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(66) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(65) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(66) InputAdapter +(68) InputAdapter Input [2]: [s_name#X, s_address#X] -(67) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [s_name#X, s_address#X] -(68) SortExecTransformer +(70) SortExecTransformer Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(69) WholeStageCodegenTransformer (X) +(71) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(70) VeloxColumnarToRowExec +(72) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(71) Scan parquet +(73) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(72) Filter +(74) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(73) Exchange +(75) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(76) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(75) Filter +(77) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(76) Scan parquet +(78) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(77) Filter +(79) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(78) Project +(80) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(79) BroadcastExchange +(81) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(80) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(81) BroadcastExchange +(83) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) Scan parquet +(84) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(83) Filter +(85) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(84) Project +(86) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(85) Scan parquet +(87) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(86) Filter +(88) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(87) Project +(89) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(88) BroadcastExchange +(90) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(90) HashAggregate +(92) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(91) Exchange +(93) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) HashAggregate +(94) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(93) Filter +(95) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(94) BroadcastHashJoin +(96) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(95) Project +(97) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(96) Exchange +(98) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(99) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(98) Project +(100) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(99) Scan parquet +(101) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(100) Filter +(102) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(101) Project +(103) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(102) BroadcastExchange +(104) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(103) BroadcastHashJoin +(105) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(104) Project +(106) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(105) Exchange +(107) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Sort +(108) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(107) AdaptiveSparkPlan +(109) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt index 4a970927299d..d22e64b92b4b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt @@ -1,85 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (92) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (59) - +- TakeOrderedAndProjectExecTransformer (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- BroadcastQueryStage (5) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- BroadcastQueryStage (14) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ FilterExecTransformer (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (60) + +- TakeOrderedAndProjectExecTransformer (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ FilterExecTransformer (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ FilterExecTransformer (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (81) - : +- BroadcastHashJoin Inner BuildRight (80) - : :- Project (75) - : : +- BroadcastHashJoin Inner BuildLeft (74) - : : :- BroadcastExchange (62) - : : : +- Filter (61) - : : : +- Scan parquet (60) - : : +- BroadcastHashJoin LeftAnti BuildRight (73) - : : :- BroadcastHashJoin LeftSemi BuildRight (68) - : : : :- Project (65) - : : : : +- Filter (64) - : : : : +- Scan parquet (63) - : : : +- BroadcastExchange (67) - : : : +- Scan parquet (66) - : : +- BroadcastExchange (72) - : : +- Project (71) - : : +- Filter (70) - : : +- Scan parquet (69) - : +- BroadcastExchange (79) - : +- Project (78) - : +- Filter (77) - : +- Scan parquet (76) - +- BroadcastExchange (85) - +- Project (84) - +- Filter (83) - +- Scan parquet (82) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- BroadcastHashJoin Inner BuildRight (87) + :- Project (82) + : +- BroadcastHashJoin Inner BuildRight (81) + : :- Project (76) + : : +- BroadcastHashJoin Inner BuildLeft (75) + : : :- BroadcastExchange (63) + : : : +- Filter (62) + : : : +- Scan parquet (61) + : : +- BroadcastHashJoin LeftAnti BuildRight (74) + : : :- BroadcastHashJoin LeftSemi BuildRight (69) + : : : :- Project (66) + : : : : +- Filter (65) + : : : : +- Scan parquet (64) + : : : +- BroadcastExchange (68) + : : : +- Scan parquet (67) + : : +- BroadcastExchange (73) + : : +- Project (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- BroadcastExchange (80) + : +- Project (79) + : +- Filter (78) + : +- Scan parquet (77) + +- BroadcastExchange (86) + +- Project (85) + +- Filter (84) + +- Scan parquet (83) (1) Scan parquet @@ -286,7 +287,7 @@ Join condition: None Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(49) FlushableHashAggregateExecTransformer +(49) RegularHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] @@ -301,194 +302,198 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(53) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [2]: [s_name#X, count#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(58) TakeOrderedAndProjectExecTransformer +(59) TakeOrderedAndProjectExecTransformer Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X], 0 -(59) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(60) Scan parquet +(61) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(61) Filter +(62) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(62) BroadcastExchange +(63) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(63) Scan parquet +(64) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(64) Filter +(65) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(65) Project +(66) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(67) BroadcastExchange +(68) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(68) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(69) Scan parquet +(70) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(70) Filter +(71) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(71) Project +(72) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(72) BroadcastExchange +(73) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(74) BroadcastHashJoin +(75) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(75) Project +(76) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(76) Scan parquet +(77) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(77) Filter +(78) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(78) Project +(79) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(79) BroadcastExchange +(80) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(80) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(81) Project +(82) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(82) Scan parquet +(83) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(83) Filter +(84) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(84) Project +(85) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(85) BroadcastExchange +(86) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(87) Project +(88) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(88) HashAggregate +(89) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(89) Exchange +(90) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(91) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(91) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(92) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt index d0252d7ac997..9344c1bf2f70 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (39) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ShuffleQueryStage (21) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (27) + +- ^ SortExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ShuffleQueryStage (22) + +- ColumnarExchange (21) + +- VeloxAppendBatches (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (37) - +- Exchange (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- BroadcastHashJoin LeftAnti BuildRight (31) - :- Filter (28) - : +- Scan parquet (27) - +- BroadcastExchange (30) - +- Scan parquet (29) + Sort (38) + +- Exchange (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- BroadcastHashJoin LeftAnti BuildRight (32) + :- Filter (29) + : +- Scan parquet (28) + +- BroadcastExchange (31) + +- Scan parquet (30) (1) Scan parquet @@ -118,87 +119,91 @@ Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacc Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) ColumnarExchange +(20) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(21) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(22) InputAdapter +(23) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) SortExecTransformer +(25) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(27) Scan parquet +(28) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(28) Filter +(29) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(29) Scan parquet +(30) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(30) BroadcastExchange +(31) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(31) BroadcastHashJoin +(32) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(32) Project +(33) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(33) HashAggregate +(34) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(34) Exchange +(35) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(36) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(36) Exchange +(37) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Sort +(38) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(38) AdaptiveSparkPlan +(39) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt index ba63341e7e03..3cd44f2a745a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- TakeOrderedAndProjectExecTransformer (33) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- BroadcastQueryStage (6) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- BroadcastQueryStage (18) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FilterExecTransformer (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (35) + +- TakeOrderedAndProjectExecTransformer (34) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ RegularHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FilterExecTransformer (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildLeft (41) - : :- BroadcastExchange (38) - : : +- Project (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + TakeOrderedAndProject (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- BroadcastHashJoin Inner BuildRight (48) + :- Project (43) + : +- BroadcastHashJoin Inner BuildLeft (42) + : :- BroadcastExchange (39) + : : +- Project (38) + : : +- Filter (37) + : : +- Scan parquet (36) + : +- Filter (41) + : +- Scan parquet (40) + +- BroadcastExchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -144,7 +145,7 @@ Join condition: None Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) FlushableHashAggregateExecTransformer +(23) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -159,131 +160,135 @@ Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: X + +(27) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(33) TakeOrderedAndProjectExecTransformer +(34) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(34) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(35) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(36) Filter +(37) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(37) Project +(38) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(38) BroadcastExchange +(39) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) Scan parquet +(40) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(42) Project +(43) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(43) Scan parquet +(44) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(45) Project +(46) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) BroadcastExchange +(47) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(48) Project +(49) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(49) HashAggregate +(50) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(50) Exchange +(51) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(52) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(52) TakeOrderedAndProject +(53) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt index 54c5c1b24d8b..703f0f47e882 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt @@ -1,43 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- BroadcastHashJoin LeftSemi BuildRight (37) - :- Project (32) - : +- Filter (31) - : +- Scan parquet (30) - +- BroadcastExchange (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (44) + +- Exchange (43) + +- HashAggregate (42) + +- Exchange (41) + +- HashAggregate (40) + +- Project (39) + +- BroadcastHashJoin LeftSemi BuildRight (38) + :- Project (33) + : +- Filter (32) + : +- Scan parquet (31) + +- BroadcastExchange (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -137,100 +138,104 @@ Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(27) SortExecTransformer +(28) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(30) Scan parquet +(31) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(31) Filter +(32) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(32) Project +(33) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(33) Scan parquet +(34) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(34) Filter +(35) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(35) Project +(36) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(36) BroadcastExchange +(37) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(37) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(38) Project +(39) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(39) HashAggregate +(40) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(40) Exchange +(41) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) HashAggregate +(42) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(42) Exchange +(43) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(44) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt index 2f037bd9c1d1..56a69c4dad35 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt @@ -1,91 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (100) +AdaptiveSparkPlan (101) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (99) - +- Exchange (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Project (71) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (92) - +- Project (91) - +- Filter (90) - +- Scan parquet (89) + Sort (100) + +- Exchange (99) + +- HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- BroadcastHashJoin Inner BuildRight (94) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Project (72) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (93) + +- Project (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -337,196 +338,200 @@ Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedpric Input [2]: [n_name#X, revenue#X] Arguments: false -(59) ColumnarExchange +(59) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(60) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(61) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(61) InputAdapter +(62) InputAdapter Input [2]: [n_name#X, revenue#X] -(62) InputIteratorTransformer +(63) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(63) SortExecTransformer +(64) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(65) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(67) Filter +(68) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(68) BroadcastExchange +(69) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(71) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(73) Project +(74) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(75) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Project +(79) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(79) Scan parquet +(80) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(81) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(81) BroadcastExchange +(82) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(83) Project +(84) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(84) Scan parquet +(85) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(86) BroadcastExchange +(87) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) Project +(89) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(89) Scan parquet +(90) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(91) Project +(92) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(92) BroadcastExchange +(93) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(93) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(94) Project +(95) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(95) HashAggregate +(96) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(96) Exchange +(97) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(98) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(98) Exchange +(99) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Sort +(100) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(100) AdaptiveSparkPlan +(101) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/6.txt index b39d8c1b2aec..41f69e74d1a0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnot Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt index 4b2aa1e644a2..5e99678cabb7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt @@ -1,85 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ SortExecTransformer (58) - +- ^ InputIteratorTransformer (57) - +- ShuffleQueryStage (55) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ FilterExecTransformer (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ FilterExecTransformer (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34) - : +- ColumnarBroadcastExchange (33) - : +- ^ FilterExecTransformer (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40) - +- ReusedExchange (39) + VeloxColumnarToRowExec (61) + +- ^ SortExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56) + +- ColumnarExchange (55) + +- VeloxAppendBatches (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ FilterExecTransformer (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ FilterExecTransformer (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40) + +- ReusedExchange (39) +- == Initial Plan == - Sort (92) - +- Exchange (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (63) - : : : : : +- Filter (62) - : : : : : +- Scan parquet (61) - : : : : +- Filter (65) - : : : : +- Scan parquet (64) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (85) - +- Filter (84) - +- Scan parquet (83) + Sort (93) + +- Exchange (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- BroadcastHashJoin Inner BuildRight (87) + :- Project (83) + : +- BroadcastHashJoin Inner BuildRight (82) + : :- Project (78) + : : +- BroadcastHashJoin Inner BuildRight (77) + : : :- Project (73) + : : : +- BroadcastHashJoin Inner BuildRight (72) + : : : :- Project (68) + : : : : +- BroadcastHashJoin Inner BuildLeft (67) + : : : : :- BroadcastExchange (64) + : : : : : +- Filter (63) + : : : : : +- Scan parquet (62) + : : : : +- Filter (66) + : : : : +- Scan parquet (65) + : : : +- BroadcastExchange (71) + : : : +- Filter (70) + : : : +- Scan parquet (69) + : : +- BroadcastExchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- BroadcastExchange (81) + : +- Filter (80) + : +- Scan parquet (79) + +- BroadcastExchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -307,188 +308,192 @@ Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) ColumnarExchange +(54) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(55) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(56) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(56) InputAdapter +(57) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(57) InputIteratorTransformer +(58) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) SortExecTransformer +(59) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(59) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(60) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(61) Scan parquet +(62) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(63) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(63) BroadcastExchange +(64) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(64) Scan parquet +(65) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(65) Filter +(66) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(66) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(67) Project +(68) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(68) Scan parquet +(69) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(69) Filter +(70) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(70) BroadcastExchange +(71) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(72) Project +(73) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(77) Project +(78) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(79) Filter +(80) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(80) BroadcastExchange +(81) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(83) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(87) Project +(88) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(88) HashAggregate +(89) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(89) Exchange +(90) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(91) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(91) Exchange +(92) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(93) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(93) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt index fe9d78579598..f008d04b81aa 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt @@ -1,116 +1,117 @@ == Physical Plan == -AdaptiveSparkPlan (129) +AdaptiveSparkPlan (130) +- == Final Plan == - VeloxColumnarToRowExec (84) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ FilterExecTransformer (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ FilterExecTransformer (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ FilterExecTransformer (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ FilterExecTransformer (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53) - : +- ColumnarBroadcastExchange (52) - : +- ^ FilterExecTransformer (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ FilterExecTransformer (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (85) + +- ^ SortExecTransformer (83) + +- ^ InputIteratorTransformer (82) + +- ShuffleQueryStage (80) + +- ColumnarExchange (79) + +- VeloxAppendBatches (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ShuffleQueryStage (72) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ FilterExecTransformer (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ FilterExecTransformer (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ FilterExecTransformer (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ FilterExecTransformer (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ FilterExecTransformer (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53) + : +- ColumnarBroadcastExchange (52) + : +- ^ FilterExecTransformer (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ FilterExecTransformer (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (128) - +- Exchange (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- BroadcastHashJoin Inner BuildRight (122) - :- Project (117) - : +- BroadcastHashJoin Inner BuildRight (116) - : :- Project (112) - : : +- BroadcastHashJoin Inner BuildRight (111) - : : :- Project (107) - : : : +- BroadcastHashJoin Inner BuildRight (106) - : : : :- Project (102) - : : : : +- BroadcastHashJoin Inner BuildRight (101) - : : : : :- Project (97) - : : : : : +- BroadcastHashJoin Inner BuildRight (96) - : : : : : :- Project (92) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) - : : : : : : :- BroadcastExchange (88) - : : : : : : : +- Project (87) - : : : : : : : +- Filter (86) - : : : : : : : +- Scan parquet (85) - : : : : : : +- Filter (90) - : : : : : : +- Scan parquet (89) - : : : : : +- BroadcastExchange (95) - : : : : : +- Filter (94) - : : : : : +- Scan parquet (93) - : : : : +- BroadcastExchange (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- BroadcastExchange (105) - : : : +- Filter (104) - : : : +- Scan parquet (103) - : : +- BroadcastExchange (110) - : : +- Filter (109) - : : +- Scan parquet (108) - : +- BroadcastExchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- BroadcastExchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (129) + +- Exchange (128) + +- HashAggregate (127) + +- Exchange (126) + +- HashAggregate (125) + +- Project (124) + +- BroadcastHashJoin Inner BuildRight (123) + :- Project (118) + : +- BroadcastHashJoin Inner BuildRight (117) + : :- Project (113) + : : +- BroadcastHashJoin Inner BuildRight (112) + : : :- Project (108) + : : : +- BroadcastHashJoin Inner BuildRight (107) + : : : :- Project (103) + : : : : +- BroadcastHashJoin Inner BuildRight (102) + : : : : :- Project (98) + : : : : : +- BroadcastHashJoin Inner BuildRight (97) + : : : : : :- Project (93) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (92) + : : : : : : :- BroadcastExchange (89) + : : : : : : : +- Project (88) + : : : : : : : +- Filter (87) + : : : : : : : +- Scan parquet (86) + : : : : : : +- Filter (91) + : : : : : : +- Scan parquet (90) + : : : : : +- BroadcastExchange (96) + : : : : : +- Filter (95) + : : : : : +- Scan parquet (94) + : : : : +- BroadcastExchange (101) + : : : : +- Filter (100) + : : : : +- Scan parquet (99) + : : : +- BroadcastExchange (106) + : : : +- Filter (105) + : : : +- Scan parquet (104) + : : +- BroadcastExchange (111) + : : +- Filter (110) + : : +- Scan parquet (109) + : +- BroadcastExchange (116) + : +- Filter (115) + : +- Scan parquet (114) + +- BroadcastExchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) (1) Scan parquet @@ -442,244 +443,248 @@ Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.000 Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) ColumnarExchange +(78) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(79) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(80) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(80) InputAdapter +(81) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(81) InputIteratorTransformer +(82) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(82) SortExecTransformer +(83) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(83) WholeStageCodegenTransformer (X) +(84) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(84) VeloxColumnarToRowExec +(85) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(85) Scan parquet +(86) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(87) Project +(88) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(88) BroadcastExchange +(89) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) Scan parquet +(90) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(92) Project +(93) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(93) Scan parquet +(94) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(94) Filter +(95) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(95) BroadcastExchange +(96) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(96) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(97) Project +(98) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(98) Scan parquet +(99) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(99) Filter +(100) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(100) BroadcastExchange +(101) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(101) BroadcastHashJoin +(102) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(102) Project +(103) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(103) Scan parquet +(104) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(104) Filter +(105) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(105) BroadcastExchange +(106) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(106) BroadcastHashJoin +(107) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(107) Project +(108) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(108) Scan parquet +(109) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(109) Filter +(110) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(110) BroadcastExchange +(111) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(111) BroadcastHashJoin +(112) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(112) Project +(113) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(113) Scan parquet +(114) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(114) Filter +(115) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(115) BroadcastExchange +(116) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(116) BroadcastHashJoin +(117) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(117) Project +(118) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(118) Scan parquet +(119) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(119) Filter +(120) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(120) Project +(121) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(121) BroadcastExchange +(122) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(122) BroadcastHashJoin +(123) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(123) Project +(124) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(124) HashAggregate +(125) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) Exchange +(126) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(127) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(127) Exchange +(128) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(129) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(129) AdaptiveSparkPlan +(130) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt index cb62cd97d3e0..dcfe2020f21c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt @@ -1,89 +1,90 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (99) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- ^ SortExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44) - +- ColumnarBroadcastExchange (43) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (65) + +- ^ SortExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ShuffleQueryStage (53) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (97) - +- Exchange (96) - +- HashAggregate (95) - +- Exchange (94) - +- HashAggregate (93) - +- Project (92) - +- BroadcastHashJoin Inner BuildRight (91) - :- Project (87) - : +- BroadcastHashJoin Inner BuildRight (86) - : :- Project (82) - : : +- BroadcastHashJoin Inner BuildRight (81) - : : :- Project (77) - : : : +- BroadcastHashJoin Inner BuildRight (76) - : : : :- Project (72) - : : : : +- BroadcastHashJoin Inner BuildLeft (71) - : : : : :- BroadcastExchange (68) - : : : : : +- Project (67) - : : : : : +- Filter (66) - : : : : : +- Scan parquet (65) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (75) - : : : +- Filter (74) - : : : +- Scan parquet (73) - : : +- BroadcastExchange (80) - : : +- Filter (79) - : : +- Scan parquet (78) - : +- BroadcastExchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- BroadcastExchange (90) - +- Filter (89) - +- Scan parquet (88) + Sort (98) + +- Exchange (97) + +- HashAggregate (96) + +- Exchange (95) + +- HashAggregate (94) + +- Project (93) + +- BroadcastHashJoin Inner BuildRight (92) + :- Project (88) + : +- BroadcastHashJoin Inner BuildRight (87) + : :- Project (83) + : : +- BroadcastHashJoin Inner BuildRight (82) + : : :- Project (78) + : : : +- BroadcastHashJoin Inner BuildRight (77) + : : : :- Project (73) + : : : : +- BroadcastHashJoin Inner BuildLeft (72) + : : : : :- BroadcastExchange (69) + : : : : : +- Project (68) + : : : : : +- Filter (67) + : : : : : +- Scan parquet (66) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (76) + : : : +- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- BroadcastExchange (86) + : +- Filter (85) + : +- Scan parquet (84) + +- BroadcastExchange (91) + +- Filter (90) + +- Scan parquet (89) (1) Scan parquet @@ -331,192 +332,196 @@ Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) ColumnarExchange +(58) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(59) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(60) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(60) InputAdapter +(61) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(61) InputIteratorTransformer +(62) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) SortExecTransformer +(63) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(63) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(64) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(67) Project +(68) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(69) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(72) Project +(73) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(77) Project +(78) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(79) Filter +(80) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(80) BroadcastExchange +(81) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(82) Project +(83) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(87) Project +(88) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(88) Scan parquet +(89) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(90) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(90) BroadcastExchange +(91) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(92) Project +(93) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(93) HashAggregate +(94) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(94) Exchange +(95) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) HashAggregate +(96) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(96) Exchange +(97) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(98) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(98) AdaptiveSparkPlan +(99) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/1.txt index 89de3133895b..2f3257bbdafc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt index 898458b34cb7..7ccdb0e04514 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt @@ -1,64 +1,65 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- BroadcastQueryStage (8), Statistics(X) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ FilterExecTransformer (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- BroadcastQueryStage (18), Statistics(X) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ FilterExecTransformer (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- BroadcastQueryStage (27), Statistics(X) - +- ColumnarBroadcastExchange (26) - +- ^ FilterExecTransformer (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (44) + +- TakeOrderedAndProjectExecTransformer (43) + +- ^ ProjectExecTransformer (41) + +- ^ RegularHashAggregateExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- VeloxAppendBatches (35) + +- ^ ProjectExecTransformer (33) + +- ^ RegularHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ FilterExecTransformer (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ FilterExecTransformer (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ FilterExecTransformer (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- BroadcastHashJoin Inner BuildRight (61) - :- Project (57) - : +- BroadcastHashJoin Inner BuildRight (56) - : :- Project (51) - : : +- BroadcastHashJoin Inner BuildRight (50) - : : :- Filter (45) - : : : +- Scan parquet (44) - : : +- BroadcastExchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (55) - : +- Project (54) - : +- Filter (53) - : +- Scan parquet (52) - +- BroadcastExchange (60) - +- Filter (59) - +- Scan parquet (58) + TakeOrderedAndProject (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- BroadcastHashJoin Inner BuildRight (62) + :- Project (58) + : +- BroadcastHashJoin Inner BuildRight (57) + : :- Project (52) + : : +- BroadcastHashJoin Inner BuildRight (51) + : : :- Filter (46) + : : : +- Scan parquet (45) + : : +- BroadcastExchange (50) + : : +- Project (49) + : : +- Filter (48) + : : +- Scan parquet (47) + : +- BroadcastExchange (56) + : +- Project (55) + : +- Filter (54) + : +- Scan parquet (53) + +- BroadcastExchange (61) + +- Filter (60) + +- Scan parquet (59) (1) Scan parquet @@ -194,7 +195,7 @@ Join condition: None Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(32) FlushableHashAggregateExecTransformer +(32) RegularHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -209,155 +210,159 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(35) ColumnarExchange +(35) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(36) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(37) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(37) InputAdapter +(38) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(38) InputIteratorTransformer +(39) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(40) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(40) ProjectExecTransformer +(41) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(41) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(43) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(43) VeloxColumnarToRowExec +(44) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(44) Scan parquet +(45) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(46) Scan parquet +(47) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(48) Project +(49) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(51) Project +(52) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(52) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(53) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(54) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(55) BroadcastExchange +(56) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(57) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(58) Scan parquet +(59) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(59) Filter +(60) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(60) BroadcastExchange +(61) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(61) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) Project +(63) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(63) HashAggregate +(64) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(64) Exchange +(65) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(66) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(66) TakeOrderedAndProject +(67) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt index 6677083c130d..619379320e1c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt @@ -1,55 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7), Statistics(X) - : +- ColumnarBroadcastExchange (6) - : +- ^ FilterExecTransformer (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17), Statistics(X) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FilterExecTransformer (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ FilterExecTransformer (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FilterExecTransformer (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (57) - +- Exchange (56) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- BroadcastHashJoin Inner BuildRight (50) - :- Project (45) - : +- BroadcastHashJoin Inner BuildRight (44) - : :- Filter (40) - : : +- Scan parquet (39) - : +- BroadcastExchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- BroadcastExchange (49) - +- Project (48) - +- Filter (47) - +- Scan parquet (46) + Sort (58) + +- Exchange (57) + +- Filter (56) + +- HashAggregate (55) + +- Exchange (54) + +- HashAggregate (53) + +- Project (52) + +- BroadcastHashJoin Inner BuildRight (51) + :- Project (46) + : +- BroadcastHashJoin Inner BuildRight (45) + : :- Filter (41) + : : +- Scan parquet (40) + : +- BroadcastExchange (44) + : +- Filter (43) + : +- Scan parquet (42) + +- BroadcastExchange (50) + +- Project (49) + +- Filter (48) + +- Scan parquet (47) (1) Scan parquet @@ -187,350 +188,354 @@ Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery su Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(34) InputAdapter +(35) InputAdapter Input [2]: [ps_partkey#X, value#X] -(35) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(36) SortExecTransformer +(37) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(38) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(39) Scan parquet +(40) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(41) Scan parquet +(42) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(42) Filter +(43) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(43) BroadcastExchange +(44) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(45) Project +(46) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(46) Scan parquet +(47) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(48) Project +(49) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(51) Project +(52) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(52) HashAggregate +(53) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(53) Exchange +(54) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(55) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(55) Filter +(56) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(56) Exchange +(57) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(58) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(58) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 30 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (99) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (82) - +- ^ ProjectExecTransformer (80) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) - :- ^ ProjectExecTransformer (66) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) - : :- ^ FilterExecTransformer (60) - : : +- ^ Scan parquet (59) - : +- ^ InputIteratorTransformer (64) - : +- BroadcastQueryStage (62), Statistics(X) - : +- ReusedExchange (61) - +- ^ InputIteratorTransformer (70) - +- BroadcastQueryStage (68), Statistics(X) - +- ReusedExchange (67) + VeloxColumnarToRowExec (83) + +- ^ ProjectExecTransformer (81) + +- ^ RegularHashAggregateExecTransformer (80) + +- ^ InputIteratorTransformer (79) + +- ShuffleQueryStage (77), Statistics(X) + +- ColumnarExchange (76) + +- ^ FlushableHashAggregateExecTransformer (74) + +- ^ ProjectExecTransformer (73) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (72) + :- ^ ProjectExecTransformer (67) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + : :- ^ FilterExecTransformer (61) + : : +- ^ Scan parquet (60) + : +- ^ InputIteratorTransformer (65) + : +- BroadcastQueryStage (63), Statistics(X) + : +- ReusedExchange (62) + +- ^ InputIteratorTransformer (71) + +- BroadcastQueryStage (69), Statistics(X) + +- ReusedExchange (68) +- == Initial Plan == - HashAggregate (98) - +- Exchange (97) - +- HashAggregate (96) - +- Project (95) - +- BroadcastHashJoin Inner BuildRight (94) - :- Project (89) - : +- BroadcastHashJoin Inner BuildRight (88) - : :- Filter (84) - : : +- Scan parquet (83) - : +- BroadcastExchange (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (93) - +- Project (92) - +- Filter (91) - +- Scan parquet (90) - - -(59) Scan parquet + HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Project (96) + +- BroadcastHashJoin Inner BuildRight (95) + :- Project (90) + : +- BroadcastHashJoin Inner BuildRight (89) + : :- Filter (85) + : : +- Scan parquet (84) + : +- BroadcastExchange (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (94) + +- Project (93) + +- Filter (92) + +- Scan parquet (91) + + +(60) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(60) FilterExecTransformer +(61) FilterExecTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: isnotnull(ps_suppkey#X) -(61) ReusedExchange [Reuses operator id: 6] +(62) ReusedExchange [Reuses operator id: 6] Output [2]: [s_suppkey#X, s_nationkey#X] -(62) BroadcastQueryStage +(63) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(63) InputAdapter +(64) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(64) InputIteratorTransformer +(65) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(65) BroadcastHashJoinExecTransformer +(66) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(66) ProjectExecTransformer +(67) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(67) ReusedExchange [Reuses operator id: 16] +(68) ReusedExchange [Reuses operator id: 16] Output [1]: [n_nationkey#X] -(68) BroadcastQueryStage +(69) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(69) InputAdapter +(70) InputAdapter Input [1]: [n_nationkey#X] -(70) InputIteratorTransformer +(71) InputIteratorTransformer Input [1]: [n_nationkey#X] -(71) BroadcastHashJoinExecTransformer +(72) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(72) ProjectExecTransformer +(73) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(73) FlushableHashAggregateExecTransformer +(74) FlushableHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(74) WholeStageCodegenTransformer (X) +(75) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(75) ColumnarExchange +(76) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(77) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(77) InputAdapter +(78) InputAdapter Input [2]: [sum#X, isEmpty#X] -(78) InputIteratorTransformer +(79) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(79) RegularHashAggregateExecTransformer +(80) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(80) ProjectExecTransformer +(81) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(81) WholeStageCodegenTransformer (X) +(82) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(82) VeloxColumnarToRowExec +(83) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(83) Scan parquet +(84) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(84) Filter +(85) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(85) Scan parquet +(86) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(87) BroadcastExchange +(88) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(89) Project +(90) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(90) Scan parquet +(91) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(91) Filter +(92) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(92) Project +(93) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(93) BroadcastExchange +(94) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(94) BroadcastHashJoin +(95) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(95) Project +(96) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(96) HashAggregate +(97) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(97) Exchange +(98) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(98) HashAggregate +(99) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(99) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt index e83423c0b2f2..efb7a9793b9e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt @@ -1,41 +1,42 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (43) +- == Final Plan == - VeloxColumnarToRowExec (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (29) + +- ^ SortExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- VeloxAppendBatches (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) + Sort (42) + +- Exchange (41) + +- HashAggregate (40) + +- Exchange (39) + +- HashAggregate (38) + +- Project (37) + +- BroadcastHashJoin Inner BuildLeft (36) + :- BroadcastExchange (32) + : +- Filter (31) + : +- Scan parquet (30) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -131,96 +132,100 @@ Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_ Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) SortExecTransformer +(27) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(27) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(30) Filter +(31) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(31) BroadcastExchange +(32) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(32) Scan parquet +(33) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(33) Filter +(34) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(34) Project +(35) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(35) BroadcastHashJoin +(36) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) Project +(37) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(37) HashAggregate +(38) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(38) Exchange +(39) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(40) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(40) Exchange +(41) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(42) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(43) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt index 1fa2d94b096b..e75208be3e05 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt @@ -1,49 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FilterExecTransformer (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ FilterExecTransformer (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- BroadcastHashJoin LeftOuter BuildRight (42) - :- Scan parquet (37) - +- BroadcastExchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- BroadcastHashJoin LeftOuter BuildRight (43) + :- Scan parquet (38) + +- BroadcastExchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -174,109 +175,113 @@ Results [2]: [c_count#X, count(1)#X AS custdist#X] Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(33) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(35) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(38) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(44) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) Exchange +(46) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(48) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(49) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(50) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(51) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(52) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt index 5bc1aef67790..332e9d086ddf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt @@ -1,35 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- BroadcastHashJoin Inner BuildRight (29) - :- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- BroadcastExchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- BroadcastHashJoin Inner BuildRight (30) + :- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- BroadcastExchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -85,7 +86,7 @@ Join condition: None Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] @@ -96,95 +97,99 @@ Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(23) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(25) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) Scan parquet +(27) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(28) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) BroadcastExchange +(29) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastHashJoin +(30) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(30) Project +(31) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(32) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) Exchange +(33) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] -(34) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt index 5cd45806c9fb..e82d3da63e53 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt @@ -1,42 +1,43 @@ == Physical Plan == -AdaptiveSparkPlan (41) +AdaptiveSparkPlan (42) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- AQEShuffleRead (25) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (27) + +- AQEShuffleRead (26) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (40) - +- Exchange (39) - +- Project (38) - +- BroadcastHashJoin Inner BuildLeft (37) - :- BroadcastExchange (29) - : +- Filter (28) - : +- Scan parquet (27) - +- Filter (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- Filter (31) - +- Scan parquet (30) + Sort (41) + +- Exchange (40) + +- Project (39) + +- BroadcastHashJoin Inner BuildLeft (38) + :- BroadcastExchange (30) + : +- Filter (29) + : +- Scan parquet (28) + +- Filter (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- Filter (32) + +- Scan parquet (31) (1) Scan parquet @@ -136,246 +137,250 @@ Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_ Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(25) AQEShuffleRead +(26) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(26) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) Scan parquet +(28) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(28) Filter +(29) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) Scan parquet +(31) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(32) Project +(33) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(33) HashAggregate +(34) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(34) Exchange +(35) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(36) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(36) Filter +(37) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(37) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(38) Project +(39) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(39) Exchange +(40) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) Sort +(41) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(41) AdaptiveSparkPlan +(42) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 19 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (66) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (57) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ ProjectExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ FilterExecTransformer (43) - +- ^ Scan parquet (42) + VeloxColumnarToRowExec (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- ^ ProjectExecTransformer (47) + +- ^ FlushableHashAggregateExecTransformer (46) + +- ^ ProjectExecTransformer (45) + +- ^ FilterExecTransformer (44) + +- ^ Scan parquet (43) +- == Initial Plan == - HashAggregate (65) - +- HashAggregate (64) - +- HashAggregate (63) - +- Exchange (62) - +- HashAggregate (61) - +- Project (60) - +- Filter (59) - +- Scan parquet (58) + HashAggregate (66) + +- HashAggregate (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- Project (61) + +- Filter (60) + +- Scan parquet (59) -(42) Scan parquet +(43) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(43) FilterExecTransformer +(44) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(45) FlushableHashAggregateExecTransformer +(46) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(46) ProjectExecTransformer +(47) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(49) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(53) ProjectExecTransformer +(54) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(54) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(55) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(56) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(57) VeloxColumnarToRowExec +(58) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(58) Scan parquet +(59) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(59) Filter +(60) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(60) Project +(61) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(61) HashAggregate +(62) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(62) Exchange +(63) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(64) HashAggregate +(65) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(65) HashAggregate +(66) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(66) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt index 56d39742c76e..83440db8a3a5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt @@ -1,53 +1,54 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (57) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ FilterExecTransformer (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ FilterExecTransformer (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- BroadcastHashJoin Inner BuildRight (46) - :- BroadcastHashJoin LeftAnti BuildRight (42) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (45) - +- Filter (44) - +- Scan parquet (43) + Sort (56) + +- Exchange (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- BroadcastHashJoin Inner BuildRight (47) + :- BroadcastHashJoin LeftAnti BuildRight (43) + : :- Filter (38) + : : +- Scan parquet (37) + : +- BroadcastExchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- BroadcastExchange (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -175,134 +176,138 @@ Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_c Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) ColumnarExchange +(29) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(30) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(36) Scan parquet +(37) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(37) Filter +(38) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(38) Scan parquet +(39) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(39) Filter +(40) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(40) Project +(41) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(43) Scan parquet +(44) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(45) BroadcastExchange +(46) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(47) Project +(48) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(48) HashAggregate +(49) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(49) Exchange +(50) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(51) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(51) HashAggregate +(52) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(52) Exchange +(53) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(54) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(56) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(56) AdaptiveSparkPlan +(57) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/17.txt index 0b41400d5864..a57c908f0178 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/17.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (14) - +- ^ ProjectExecTransformer (12) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- RowToVeloxColumnar (2) - +- LocalTableScan (1) + VeloxColumnarToRowExec (15) + +- ^ ProjectExecTransformer (13) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ RegularHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == - HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin Inner BuildRight (30) - :- Project (22) - : +- BroadcastHashJoin Inner BuildRight (21) - : :- Filter (16) - : : +- Scan parquet (15) - : +- BroadcastExchange (20) - : +- Project (19) - : +- Filter (18) - : +- Scan parquet (17) - +- BroadcastExchange (29) - +- Filter (28) - +- HashAggregate (27) - +- Exchange (26) - +- HashAggregate (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin Inner BuildRight (31) + :- Project (23) + : +- BroadcastHashJoin Inner BuildRight (22) + : :- Filter (17) + : : +- Scan parquet (16) + : +- BroadcastExchange (21) + : +- Project (20) + : +- Filter (19) + : +- Scan parquet (18) + +- BroadcastExchange (30) + +- Filter (29) + +- HashAggregate (28) + +- Exchange (27) + +- HashAggregate (26) + +- Filter (25) + +- Scan parquet (24) (1) LocalTableScan @@ -47,7 +48,7 @@ Input [1]: [l_extendedprice#X] (4) InputIteratorTransformer Input [1]: [l_extendedprice#X] -(5) FlushableHashAggregateExecTransformer +(5) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] @@ -58,141 +59,145 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(8) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [2]: [sum#X, isEmpty#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(14) VeloxColumnarToRowExec +(15) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(15) Scan parquet +(16) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(16) Filter +(17) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(17) Scan parquet +(18) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(18) Filter +(19) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(19) Project +(20) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(20) BroadcastExchange +(21) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastHashJoin +(22) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(22) Project +(23) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(23) Scan parquet +(24) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(25) HashAggregate +(26) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(26) Exchange +(27) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) HashAggregate +(28) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(28) Filter +(29) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(31) Project +(32) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(32) HashAggregate +(33) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] -(35) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt index c37f0b49567c..a656d4e5433c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt @@ -1,80 +1,82 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- TakeOrderedAndProjectExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ FilterExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23), Statistics(X) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- BroadcastQueryStage (32), Statistics(X) - +- ReusedExchange (31) + VeloxColumnarToRowExec (55) + +- TakeOrderedAndProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- VeloxAppendBatches (47) + +- ^ ProjectExecTransformer (45) + +- ^ RegularHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) + :- ^ ProjectExecTransformer (29) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (28) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (27) + : :- ^ FilterExecTransformer (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (26) + : +- BroadcastQueryStage (24), Statistics(X) + : +- ColumnarBroadcastExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FilterExecTransformer (20) + : +- ^ RegularHashAggregateExecTransformer (19) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ RegularHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (41) + +- BroadcastQueryStage (39), Statistics(X) + +- ColumnarBroadcastExchange (38) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (36) + :- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (35) + +- BroadcastQueryStage (33), Statistics(X) + +- ReusedExchange (32) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (68) - : +- BroadcastHashJoin Inner BuildLeft (67) - : :- BroadcastExchange (56) - : : +- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastHashJoin LeftSemi BuildRight (66) - : :- Filter (58) - : : +- Scan parquet (57) - : +- BroadcastExchange (65) - : +- Project (64) - : +- Filter (63) - : +- HashAggregate (62) - : +- Exchange (61) - : +- HashAggregate (60) - : +- Scan parquet (59) - +- BroadcastExchange (79) - +- BroadcastHashJoin LeftSemi BuildRight (78) - :- Filter (70) - : +- Scan parquet (69) - +- BroadcastExchange (77) - +- Project (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Scan parquet (71) + TakeOrderedAndProject (87) + +- HashAggregate (86) + +- Exchange (85) + +- HashAggregate (84) + +- Project (83) + +- BroadcastHashJoin Inner BuildRight (82) + :- Project (70) + : +- BroadcastHashJoin Inner BuildLeft (69) + : :- BroadcastExchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- BroadcastHashJoin LeftSemi BuildRight (68) + : :- Filter (60) + : : +- Scan parquet (59) + : +- BroadcastExchange (67) + : +- Project (66) + : +- Filter (65) + : +- HashAggregate (64) + : +- Exchange (63) + : +- HashAggregate (62) + : +- Scan parquet (61) + +- BroadcastExchange (81) + +- BroadcastHashJoin LeftSemi BuildRight (80) + :- Filter (72) + : +- Scan parquet (71) + +- BroadcastExchange (79) + +- Project (78) + +- Filter (77) + +- HashAggregate (76) + +- Exchange (75) + +- HashAggregate (74) + +- Scan parquet (73) (1) Scan parquet @@ -123,7 +125,7 @@ Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) FlushableHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] @@ -138,333 +140,341 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [1]: [l_orderkey#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [1]: [l_orderkey#X] -(26) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(27) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(30) FilterExecTransformer +(31) FilterExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Arguments: isnotnull(l_orderkey#X) -(31) ReusedExchange [Reuses operator id: 22] +(32) ReusedExchange [Reuses operator id: 23] Output [1]: [l_orderkey#X] -(32) BroadcastQueryStage +(33) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [1]: [l_orderkey#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [l_orderkey#X] -(35) BroadcastHashJoinExecTransformer +(36) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(37) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(39) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(41) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(43) FlushableHashAggregateExecTransformer +(44) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(47) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: X + +(48) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(49) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(50) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(51) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(51) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(52) TakeOrderedAndProjectExecTransformer +(54) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(53) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(54) Scan parquet +(56) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(56) BroadcastExchange +(58) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(59) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(58) Filter +(60) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(59) Scan parquet +(61) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(60) HashAggregate +(62) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(61) Exchange +(63) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(63) Filter +(65) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(64) Project +(66) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(65) BroadcastExchange +(67) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(67) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(68) Project +(70) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(69) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(71) Scan parquet +(73) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(74) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(75) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(76) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(77) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(78) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) BroadcastExchange +(79) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(79) BroadcastExchange +(81) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(81) Project +(83) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(82) HashAggregate +(84) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(83) Exchange +(85) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(86) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(85) TakeOrderedAndProject +(87) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(86) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt index 6822887d3aed..8cc8b37b9909 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt @@ -1,34 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (32) - +- Exchange (31) - +- HashAggregate (30) - +- Project (29) - +- BroadcastHashJoin Inner BuildRight (28) - :- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- BroadcastExchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -84,7 +85,7 @@ Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -95,91 +96,95 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(20) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(22) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(23) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(24) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(25) Scan parquet +(26) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(27) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) BroadcastExchange +(28) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(30) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(31) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) Exchange +(32) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(32) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(33) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt index 32f28a0311c4..afee0913dbde 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt @@ -1,96 +1,98 @@ == Physical Plan == -AdaptiveSparkPlan (104) +AdaptiveSparkPlan (106) +- == Final Plan == - VeloxColumnarToRowExec (67) - +- AQEShuffleRead (66) - +- ShuffleQueryStage (65), Statistics(X) - +- ColumnarExchange (64) - +- ^ ProjectExecTransformer (62) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (61) - :- ^ ProjectExecTransformer (52) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (51) - : :- ^ InputIteratorTransformer (9) - : : +- AQEShuffleRead (7) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (50) - : +- BroadcastQueryStage (48), Statistics(X) - : +- ColumnarBroadcastExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) - : :- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) - : : :- ^ FilterExecTransformer (11) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (19) - : : +- BroadcastQueryStage (17), Statistics(X) - : : +- ColumnarBroadcastExchange (16) - : : +- ^ ProjectExecTransformer (14) - : : +- ^ FilterExecTransformer (13) - : : +- ^ Scan parquet (12) - : +- ^ FilterExecTransformer (43) - : +- ^ ProjectExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ InputIteratorTransformer (40) - : +- ShuffleQueryStage (38), Statistics(X) - : +- ColumnarExchange (37) - : +- ^ ProjectExecTransformer (35) - : +- ^ FlushableHashAggregateExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) - : :- ^ ProjectExecTransformer (28) - : : +- ^ FilterExecTransformer (27) - : : +- ^ Scan parquet (26) - : +- ^ InputIteratorTransformer (32) - : +- BroadcastQueryStage (30), Statistics(X) - : +- ReusedExchange (29) - +- ^ InputIteratorTransformer (60) - +- BroadcastQueryStage (58), Statistics(X) - +- ColumnarBroadcastExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ FilterExecTransformer (54) - +- ^ Scan parquet (53) + VeloxColumnarToRowExec (69) + +- AQEShuffleRead (68) + +- ShuffleQueryStage (67), Statistics(X) + +- ColumnarExchange (66) + +- VeloxAppendBatches (65) + +- ^ ProjectExecTransformer (63) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (62) + :- ^ ProjectExecTransformer (53) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (52) + : :- ^ InputIteratorTransformer (10) + : : +- AQEShuffleRead (8) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (51) + : +- BroadcastQueryStage (49), Statistics(X) + : +- ColumnarBroadcastExchange (48) + : +- ^ ProjectExecTransformer (46) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (45) + : :- ^ InputIteratorTransformer (26) + : : +- BroadcastQueryStage (24), Statistics(X) + : : +- ColumnarBroadcastExchange (23) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (21) + : : :- ^ FilterExecTransformer (12) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (20) + : : +- BroadcastQueryStage (18), Statistics(X) + : : +- ColumnarBroadcastExchange (17) + : : +- ^ ProjectExecTransformer (15) + : : +- ^ FilterExecTransformer (14) + : : +- ^ Scan parquet (13) + : +- ^ FilterExecTransformer (44) + : +- ^ ProjectExecTransformer (43) + : +- ^ RegularHashAggregateExecTransformer (42) + : +- ^ InputIteratorTransformer (41) + : +- ShuffleQueryStage (39), Statistics(X) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ FlushableHashAggregateExecTransformer (35) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (34) + : :- ^ ProjectExecTransformer (29) + : : +- ^ FilterExecTransformer (28) + : : +- ^ Scan parquet (27) + : +- ^ InputIteratorTransformer (33) + : +- BroadcastQueryStage (31), Statistics(X) + : +- ReusedExchange (30) + +- ^ InputIteratorTransformer (61) + +- BroadcastQueryStage (59), Statistics(X) + +- ColumnarBroadcastExchange (58) + +- ^ ProjectExecTransformer (56) + +- ^ FilterExecTransformer (55) + +- ^ Scan parquet (54) +- == Initial Plan == - Sort (103) - +- Exchange (102) - +- Project (101) - +- BroadcastHashJoin Inner BuildRight (100) - :- Project (95) - : +- ShuffledHashJoin LeftSemi BuildRight (94) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Exchange (93) - : +- Project (92) - : +- BroadcastHashJoin Inner BuildLeft (91) - : :- BroadcastExchange (78) - : : +- BroadcastHashJoin LeftSemi BuildRight (77) - : : :- Filter (72) - : : : +- Scan parquet (71) - : : +- BroadcastExchange (76) - : : +- Project (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- Filter (90) - : +- HashAggregate (89) - : +- Exchange (88) - : +- HashAggregate (87) - : +- BroadcastHashJoin LeftSemi BuildRight (86) - : :- Project (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (85) - : +- Project (84) - : +- Filter (83) - : +- Scan parquet (82) - +- BroadcastExchange (99) - +- Project (98) - +- Filter (97) - +- Scan parquet (96) + Sort (105) + +- Exchange (104) + +- Project (103) + +- BroadcastHashJoin Inner BuildRight (102) + :- Project (97) + : +- ShuffledHashJoin LeftSemi BuildRight (96) + : :- Exchange (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (95) + : +- Project (94) + : +- BroadcastHashJoin Inner BuildLeft (93) + : :- BroadcastExchange (80) + : : +- BroadcastHashJoin LeftSemi BuildRight (79) + : : :- Filter (74) + : : : +- Scan parquet (73) + : : +- BroadcastExchange (78) + : : +- Project (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- Filter (92) + : +- HashAggregate (91) + : +- Exchange (90) + : +- HashAggregate (89) + : +- BroadcastHashJoin LeftSemi BuildRight (88) + : :- Project (83) + : : +- Filter (82) + : : +- Scan parquet (81) + : +- BroadcastExchange (87) + : +- Project (86) + : +- Filter (85) + : +- Scan parquet (84) + +- BroadcastExchange (101) + +- Project (100) + +- Filter (99) + +- Scan parquet (98) (1) Scan parquet @@ -112,438 +114,446 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) AQEShuffleRead +(8) AQEShuffleRead Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: local -(8) InputAdapter +(9) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(10) Scan parquet +(11) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(11) FilterExecTransformer +(12) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(12) Scan parquet +(13) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(13) FilterExecTransformer +(14) FilterExecTransformer Input [2]: [p_partkey#X, p_name#X] Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(14) ProjectExecTransformer +(15) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(15) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(16) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(17) BroadcastQueryStage +(18) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [1]: [p_partkey#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [1]: [p_partkey#X] -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(26) Scan parquet +(27) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(27) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(29) ReusedExchange [Reuses operator id: 16] +(30) ReusedExchange [Reuses operator id: 17] Output [1]: [p_partkey#X] -(30) BroadcastQueryStage +(31) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [1]: [p_partkey#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [1]: [p_partkey#X] -(33) BroadcastHashJoinExecTransformer +(34) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(34) FlushableHashAggregateExecTransformer +(35) FlushableHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(36) ProjectExecTransformer Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(38) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(39) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(42) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) FilterExecTransformer +(44) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(44) BroadcastHashJoinExecTransformer +(45) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(45) ProjectExecTransformer +(46) ProjectExecTransformer Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(46) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] Arguments: false -(47) ColumnarBroadcastExchange +(48) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(48) BroadcastQueryStage +(49) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(49) InputAdapter +(50) InputAdapter Input [1]: [ps_suppkey#X] -(50) InputIteratorTransformer +(51) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(51) BroadcastHashJoinExecTransformer +(52) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(52) ProjectExecTransformer +(53) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(53) Scan parquet +(54) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(54) FilterExecTransformer +(55) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(55) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(56) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(57) ColumnarBroadcastExchange +(58) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(58) BroadcastQueryStage +(59) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(59) InputAdapter +(60) InputAdapter Input [1]: [n_nationkey#X] -(60) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [n_nationkey#X] -(61) BroadcastHashJoinExecTransformer +(62) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) ProjectExecTransformer +(63) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(63) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(64) ColumnarExchange +(65) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(66) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(65) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(66) AQEShuffleRead +(68) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(67) VeloxColumnarToRowExec +(69) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(68) Scan parquet +(70) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(69) Filter +(71) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(70) Exchange +(72) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(73) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(72) Filter +(74) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(73) Scan parquet +(75) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(74) Filter +(76) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(75) Project +(77) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(76) BroadcastExchange +(78) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(77) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(78) BroadcastExchange +(80) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(79) Scan parquet +(81) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(80) Filter +(82) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(81) Project +(83) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(82) Scan parquet +(84) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(83) Filter +(85) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(84) Project +(86) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(85) BroadcastExchange +(87) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(86) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(87) HashAggregate +(89) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(88) Exchange +(90) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(91) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(90) Filter +(92) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(91) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(92) Project +(94) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(93) Exchange +(95) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(95) Project +(97) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(96) Scan parquet +(98) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(97) Filter +(99) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(98) Project +(100) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(99) BroadcastExchange +(101) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(100) BroadcastHashJoin +(102) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(101) Project +(103) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(102) Exchange +(104) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) Sort +(105) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(104) AdaptiveSparkPlan +(106) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt index 252a06b11fa7..d34e59327a67 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt @@ -1,84 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (91) +AdaptiveSparkPlan (92) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- BroadcastQueryStage (5), Statistics(X) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ FilterExecTransformer (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5), Statistics(X) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ FilterExecTransformer (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ FilterExecTransformer (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (90) - +- HashAggregate (89) - +- Exchange (88) - +- HashAggregate (87) - +- Project (86) - +- BroadcastHashJoin Inner BuildRight (85) - :- Project (80) - : +- BroadcastHashJoin Inner BuildRight (79) - : :- Project (74) - : : +- BroadcastHashJoin Inner BuildLeft (73) - : : :- BroadcastExchange (61) - : : : +- Filter (60) - : : : +- Scan parquet (59) - : : +- BroadcastHashJoin LeftAnti BuildRight (72) - : : :- BroadcastHashJoin LeftSemi BuildRight (67) - : : : :- Project (64) - : : : : +- Filter (63) - : : : : +- Scan parquet (62) - : : : +- BroadcastExchange (66) - : : : +- Scan parquet (65) - : : +- BroadcastExchange (71) - : : +- Project (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- BroadcastExchange (78) - : +- Project (77) - : +- Filter (76) - : +- Scan parquet (75) - +- BroadcastExchange (84) - +- Project (83) - +- Filter (82) - +- Scan parquet (81) + TakeOrderedAndProject (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (81) + : +- BroadcastHashJoin Inner BuildRight (80) + : :- Project (75) + : : +- BroadcastHashJoin Inner BuildLeft (74) + : : :- BroadcastExchange (62) + : : : +- Filter (61) + : : : +- Scan parquet (60) + : : +- BroadcastHashJoin LeftAnti BuildRight (73) + : : :- BroadcastHashJoin LeftSemi BuildRight (68) + : : : :- Project (65) + : : : : +- Filter (64) + : : : : +- Scan parquet (63) + : : : +- BroadcastExchange (67) + : : : +- Scan parquet (66) + : : +- BroadcastExchange (72) + : : +- Project (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- BroadcastExchange (79) + : +- Project (78) + : +- Filter (77) + : +- Scan parquet (76) + +- BroadcastExchange (85) + +- Project (84) + +- Filter (83) + +- Scan parquet (82) (1) Scan parquet @@ -285,7 +286,7 @@ Join condition: None Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(49) FlushableHashAggregateExecTransformer +(49) RegularHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] @@ -300,190 +301,194 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(53) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [2]: [s_name#X, count#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(58) VeloxColumnarToRowExec +(59) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(59) Scan parquet +(60) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(60) Filter +(61) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(61) BroadcastExchange +(62) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(62) Scan parquet +(63) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(63) Filter +(64) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(64) Project +(65) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(66) BroadcastExchange +(67) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(67) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(68) Scan parquet +(69) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(69) Filter +(70) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(70) Project +(71) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(71) BroadcastExchange +(72) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(74) Project +(75) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(75) Scan parquet +(76) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(76) Filter +(77) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(77) Project +(78) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(78) BroadcastExchange +(79) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(79) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(80) Project +(81) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(81) Scan parquet +(82) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(82) Filter +(83) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(83) Project +(84) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(84) BroadcastExchange +(85) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(85) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(86) Project +(87) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(87) HashAggregate +(88) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(88) Exchange +(89) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(90) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(90) TakeOrderedAndProject +(91) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(91) AdaptiveSparkPlan +(92) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt index 5a6f0dc8fe05..bf6c22d0d8b8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (39) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (27) + +- ^ SortExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- VeloxAppendBatches (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (37) - +- Exchange (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- BroadcastHashJoin LeftAnti BuildRight (31) - :- Filter (28) - : +- Scan parquet (27) - +- BroadcastExchange (30) - +- Scan parquet (29) + Sort (38) + +- Exchange (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- BroadcastHashJoin LeftAnti BuildRight (32) + :- Filter (29) + : +- Scan parquet (28) + +- BroadcastExchange (31) + +- Scan parquet (30) (1) Scan parquet @@ -118,221 +119,225 @@ Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacc Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) ColumnarExchange +(20) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(21) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(22) InputAdapter +(23) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) SortExecTransformer +(25) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(27) Scan parquet +(28) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(28) Filter +(29) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(29) Scan parquet +(30) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(30) BroadcastExchange +(31) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(31) BroadcastHashJoin +(32) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(32) Project +(33) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(33) HashAggregate +(34) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(34) Exchange +(35) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(36) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(36) Exchange +(37) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Sort +(38) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(38) AdaptiveSparkPlan +(39) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (58) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ RegularHashAggregateExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (51) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ InputIteratorTransformer (48) + +- ShuffleQueryStage (46), Statistics(X) + +- ColumnarExchange (45) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) + HashAggregate (57) + +- Exchange (56) + +- HashAggregate (55) + +- Project (54) + +- Filter (53) + +- Scan parquet (52) -(39) Scan parquet +(40) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(40) FilterExecTransformer +(41) FilterExecTransformer Input [2]: [c_phone#X, c_acctbal#X] Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(41) ProjectExecTransformer +(42) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(42) FlushableHashAggregateExecTransformer +(43) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(43) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(44) ColumnarExchange +(45) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(46) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(46) InputAdapter +(47) InputAdapter Input [2]: [sum#X, count#X] -(47) InputIteratorTransformer +(48) InputIteratorTransformer Input [2]: [sum#X, count#X] -(48) RegularHashAggregateExecTransformer +(49) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(49) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(50) VeloxColumnarToRowExec +(51) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(51) Scan parquet +(52) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(52) Filter +(53) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(53) Project +(54) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(54) HashAggregate +(55) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(55) Exchange +(56) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(57) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) AdaptiveSparkPlan +(58) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (58) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ RegularHashAggregateExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (51) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ InputIteratorTransformer (48) + +- ShuffleQueryStage (46), Statistics(X) + +- ColumnarExchange (45) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) \ No newline at end of file + HashAggregate (57) + +- Exchange (56) + +- HashAggregate (55) + +- Project (54) + +- Filter (53) + +- Scan parquet (52) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt index 722def52f850..5d022b4a9709 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- TakeOrderedAndProjectExecTransformer (33) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- BroadcastQueryStage (18), Statistics(X) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FilterExecTransformer (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (35) + +- TakeOrderedAndProjectExecTransformer (34) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ RegularHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FilterExecTransformer (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildLeft (41) - : :- BroadcastExchange (38) - : : +- Project (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + TakeOrderedAndProject (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- BroadcastHashJoin Inner BuildRight (48) + :- Project (43) + : +- BroadcastHashJoin Inner BuildLeft (42) + : :- BroadcastExchange (39) + : : +- Project (38) + : : +- Filter (37) + : : +- Scan parquet (36) + : +- Filter (41) + : +- Scan parquet (40) + +- BroadcastExchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -144,7 +145,7 @@ Join condition: None Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) FlushableHashAggregateExecTransformer +(23) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -159,131 +160,135 @@ Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: X + +(27) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(33) TakeOrderedAndProjectExecTransformer +(34) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(34) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(35) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(36) Filter +(37) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(37) Project +(38) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(38) BroadcastExchange +(39) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) Scan parquet +(40) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(42) Project +(43) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(43) Scan parquet +(44) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(45) Project +(46) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) BroadcastExchange +(47) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(48) Project +(49) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(49) HashAggregate +(50) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(50) Exchange +(51) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(52) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(52) TakeOrderedAndProject +(53) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt index 0ca16a6174b7..577a71bdeff7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt @@ -1,43 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9), Statistics(X) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- BroadcastHashJoin LeftSemi BuildRight (37) - :- Project (32) - : +- Filter (31) - : +- Scan parquet (30) - +- BroadcastExchange (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (44) + +- Exchange (43) + +- HashAggregate (42) + +- Exchange (41) + +- HashAggregate (40) + +- Project (39) + +- BroadcastHashJoin LeftSemi BuildRight (38) + :- Project (33) + : +- Filter (32) + : +- Scan parquet (31) + +- BroadcastExchange (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -137,100 +138,104 @@ Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(27) SortExecTransformer +(28) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(30) Scan parquet +(31) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(31) Filter +(32) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(32) Project +(33) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(33) Scan parquet +(34) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(34) Filter +(35) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(35) Project +(36) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(36) BroadcastExchange +(37) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(37) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(38) Project +(39) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(39) HashAggregate +(40) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(40) Exchange +(41) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) HashAggregate +(42) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(42) Exchange +(43) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(44) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt index 24ba48c495f6..189b67f8e353 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt @@ -1,91 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (100) +AdaptiveSparkPlan (101) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45), Statistics(X) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61), Statistics(X) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (99) - +- Exchange (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Project (71) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (92) - +- Project (91) - +- Filter (90) - +- Scan parquet (89) + Sort (100) + +- Exchange (99) + +- HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- BroadcastHashJoin Inner BuildRight (94) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Project (72) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (93) + +- Project (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -337,196 +338,200 @@ Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedpric Input [2]: [n_name#X, revenue#X] Arguments: false -(59) ColumnarExchange +(59) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(60) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(61) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(61) InputAdapter +(62) InputAdapter Input [2]: [n_name#X, revenue#X] -(62) InputIteratorTransformer +(63) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(63) SortExecTransformer +(64) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(65) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(67) Filter +(68) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(68) BroadcastExchange +(69) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(71) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(73) Project +(74) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(75) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Project +(79) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(79) Scan parquet +(80) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(81) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(81) BroadcastExchange +(82) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(83) Project +(84) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(84) Scan parquet +(85) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(86) BroadcastExchange +(87) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) Project +(89) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(89) Scan parquet +(90) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(91) Project +(92) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(92) BroadcastExchange +(93) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(93) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(94) Project +(95) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(95) HashAggregate +(96) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(96) Exchange +(97) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(98) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(98) Exchange +(99) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Sort +(100) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(100) AdaptiveSparkPlan +(101) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/6.txt index 68854bdea473..3a3148c1c113 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnot Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt index fd1e1e8fa37f..88226758f103 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt @@ -1,85 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ SortExecTransformer (58) - +- ^ InputIteratorTransformer (57) - +- ShuffleQueryStage (55), Statistics(X) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16), Statistics(X) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ FilterExecTransformer (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25), Statistics(X) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ FilterExecTransformer (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ FilterExecTransformer (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40), Statistics(X) - +- ReusedExchange (39) + VeloxColumnarToRowExec (61) + +- ^ SortExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- VeloxAppendBatches (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ FilterExecTransformer (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ FilterExecTransformer (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == - Sort (92) - +- Exchange (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (63) - : : : : : +- Filter (62) - : : : : : +- Scan parquet (61) - : : : : +- Filter (65) - : : : : +- Scan parquet (64) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (85) - +- Filter (84) - +- Scan parquet (83) + Sort (93) + +- Exchange (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- BroadcastHashJoin Inner BuildRight (87) + :- Project (83) + : +- BroadcastHashJoin Inner BuildRight (82) + : :- Project (78) + : : +- BroadcastHashJoin Inner BuildRight (77) + : : :- Project (73) + : : : +- BroadcastHashJoin Inner BuildRight (72) + : : : :- Project (68) + : : : : +- BroadcastHashJoin Inner BuildLeft (67) + : : : : :- BroadcastExchange (64) + : : : : : +- Filter (63) + : : : : : +- Scan parquet (62) + : : : : +- Filter (66) + : : : : +- Scan parquet (65) + : : : +- BroadcastExchange (71) + : : : +- Filter (70) + : : : +- Scan parquet (69) + : : +- BroadcastExchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- BroadcastExchange (81) + : +- Filter (80) + : +- Scan parquet (79) + +- BroadcastExchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -307,188 +308,192 @@ Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) ColumnarExchange +(54) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(55) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(56) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(56) InputAdapter +(57) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(57) InputIteratorTransformer +(58) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) SortExecTransformer +(59) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(59) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(60) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(61) Scan parquet +(62) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(63) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(63) BroadcastExchange +(64) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(64) Scan parquet +(65) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(65) Filter +(66) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(66) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(67) Project +(68) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(68) Scan parquet +(69) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(69) Filter +(70) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(70) BroadcastExchange +(71) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(72) Project +(73) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(77) Project +(78) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(79) Filter +(80) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(80) BroadcastExchange +(81) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(83) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(87) Project +(88) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(88) HashAggregate +(89) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(89) Exchange +(90) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(91) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(91) Exchange +(92) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(93) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(93) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt index fc7a79a5b2b3..e0c8a43893bd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt @@ -1,116 +1,117 @@ == Physical Plan == -AdaptiveSparkPlan (129) +AdaptiveSparkPlan (130) +- == Final Plan == - VeloxColumnarToRowExec (84) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ FilterExecTransformer (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26), Statistics(X) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ FilterExecTransformer (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35), Statistics(X) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ FilterExecTransformer (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44), Statistics(X) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ FilterExecTransformer (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53), Statistics(X) - : +- ColumnarBroadcastExchange (52) - : +- ^ FilterExecTransformer (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63), Statistics(X) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ FilterExecTransformer (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (85) + +- ^ SortExecTransformer (83) + +- ^ InputIteratorTransformer (82) + +- ShuffleQueryStage (80), Statistics(X) + +- ColumnarExchange (79) + +- VeloxAppendBatches (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ShuffleQueryStage (72), Statistics(X) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ FilterExecTransformer (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ FilterExecTransformer (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ FilterExecTransformer (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ FilterExecTransformer (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ FilterExecTransformer (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ FilterExecTransformer (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ FilterExecTransformer (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (128) - +- Exchange (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- BroadcastHashJoin Inner BuildRight (122) - :- Project (117) - : +- BroadcastHashJoin Inner BuildRight (116) - : :- Project (112) - : : +- BroadcastHashJoin Inner BuildRight (111) - : : :- Project (107) - : : : +- BroadcastHashJoin Inner BuildRight (106) - : : : :- Project (102) - : : : : +- BroadcastHashJoin Inner BuildRight (101) - : : : : :- Project (97) - : : : : : +- BroadcastHashJoin Inner BuildRight (96) - : : : : : :- Project (92) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) - : : : : : : :- BroadcastExchange (88) - : : : : : : : +- Project (87) - : : : : : : : +- Filter (86) - : : : : : : : +- Scan parquet (85) - : : : : : : +- Filter (90) - : : : : : : +- Scan parquet (89) - : : : : : +- BroadcastExchange (95) - : : : : : +- Filter (94) - : : : : : +- Scan parquet (93) - : : : : +- BroadcastExchange (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- BroadcastExchange (105) - : : : +- Filter (104) - : : : +- Scan parquet (103) - : : +- BroadcastExchange (110) - : : +- Filter (109) - : : +- Scan parquet (108) - : +- BroadcastExchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- BroadcastExchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (129) + +- Exchange (128) + +- HashAggregate (127) + +- Exchange (126) + +- HashAggregate (125) + +- Project (124) + +- BroadcastHashJoin Inner BuildRight (123) + :- Project (118) + : +- BroadcastHashJoin Inner BuildRight (117) + : :- Project (113) + : : +- BroadcastHashJoin Inner BuildRight (112) + : : :- Project (108) + : : : +- BroadcastHashJoin Inner BuildRight (107) + : : : :- Project (103) + : : : : +- BroadcastHashJoin Inner BuildRight (102) + : : : : :- Project (98) + : : : : : +- BroadcastHashJoin Inner BuildRight (97) + : : : : : :- Project (93) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (92) + : : : : : : :- BroadcastExchange (89) + : : : : : : : +- Project (88) + : : : : : : : +- Filter (87) + : : : : : : : +- Scan parquet (86) + : : : : : : +- Filter (91) + : : : : : : +- Scan parquet (90) + : : : : : +- BroadcastExchange (96) + : : : : : +- Filter (95) + : : : : : +- Scan parquet (94) + : : : : +- BroadcastExchange (101) + : : : : +- Filter (100) + : : : : +- Scan parquet (99) + : : : +- BroadcastExchange (106) + : : : +- Filter (105) + : : : +- Scan parquet (104) + : : +- BroadcastExchange (111) + : : +- Filter (110) + : : +- Scan parquet (109) + : +- BroadcastExchange (116) + : +- Filter (115) + : +- Scan parquet (114) + +- BroadcastExchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) (1) Scan parquet @@ -442,244 +443,248 @@ Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.000 Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) ColumnarExchange +(78) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(79) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(80) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(80) InputAdapter +(81) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(81) InputIteratorTransformer +(82) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(82) SortExecTransformer +(83) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(83) WholeStageCodegenTransformer (X) +(84) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(84) VeloxColumnarToRowExec +(85) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(85) Scan parquet +(86) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(87) Project +(88) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(88) BroadcastExchange +(89) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) Scan parquet +(90) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(92) Project +(93) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(93) Scan parquet +(94) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(94) Filter +(95) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(95) BroadcastExchange +(96) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(96) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(97) Project +(98) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(98) Scan parquet +(99) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(99) Filter +(100) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(100) BroadcastExchange +(101) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(101) BroadcastHashJoin +(102) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(102) Project +(103) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(103) Scan parquet +(104) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(104) Filter +(105) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(105) BroadcastExchange +(106) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(106) BroadcastHashJoin +(107) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(107) Project +(108) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(108) Scan parquet +(109) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(109) Filter +(110) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(110) BroadcastExchange +(111) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(111) BroadcastHashJoin +(112) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(112) Project +(113) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(113) Scan parquet +(114) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(114) Filter +(115) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(115) BroadcastExchange +(116) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(116) BroadcastHashJoin +(117) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(117) Project +(118) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(118) Scan parquet +(119) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(119) Filter +(120) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(120) Project +(121) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(121) BroadcastExchange +(122) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(122) BroadcastHashJoin +(123) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(123) Project +(124) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(124) HashAggregate +(125) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) Exchange +(126) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(127) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(127) Exchange +(128) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(129) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(129) AdaptiveSparkPlan +(130) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt index 4f18dfa35261..75b883057963 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt @@ -1,89 +1,90 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (99) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- ^ SortExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (65) + +- ^ SortExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (97) - +- Exchange (96) - +- HashAggregate (95) - +- Exchange (94) - +- HashAggregate (93) - +- Project (92) - +- BroadcastHashJoin Inner BuildRight (91) - :- Project (87) - : +- BroadcastHashJoin Inner BuildRight (86) - : :- Project (82) - : : +- BroadcastHashJoin Inner BuildRight (81) - : : :- Project (77) - : : : +- BroadcastHashJoin Inner BuildRight (76) - : : : :- Project (72) - : : : : +- BroadcastHashJoin Inner BuildLeft (71) - : : : : :- BroadcastExchange (68) - : : : : : +- Project (67) - : : : : : +- Filter (66) - : : : : : +- Scan parquet (65) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (75) - : : : +- Filter (74) - : : : +- Scan parquet (73) - : : +- BroadcastExchange (80) - : : +- Filter (79) - : : +- Scan parquet (78) - : +- BroadcastExchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- BroadcastExchange (90) - +- Filter (89) - +- Scan parquet (88) + Sort (98) + +- Exchange (97) + +- HashAggregate (96) + +- Exchange (95) + +- HashAggregate (94) + +- Project (93) + +- BroadcastHashJoin Inner BuildRight (92) + :- Project (88) + : +- BroadcastHashJoin Inner BuildRight (87) + : :- Project (83) + : : +- BroadcastHashJoin Inner BuildRight (82) + : : :- Project (78) + : : : +- BroadcastHashJoin Inner BuildRight (77) + : : : :- Project (73) + : : : : +- BroadcastHashJoin Inner BuildLeft (72) + : : : : :- BroadcastExchange (69) + : : : : : +- Project (68) + : : : : : +- Filter (67) + : : : : : +- Scan parquet (66) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (76) + : : : +- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- BroadcastExchange (86) + : +- Filter (85) + : +- Scan parquet (84) + +- BroadcastExchange (91) + +- Filter (90) + +- Scan parquet (89) (1) Scan parquet @@ -331,192 +332,196 @@ Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) ColumnarExchange +(58) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(59) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(60) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(60) InputAdapter +(61) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(61) InputIteratorTransformer +(62) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) SortExecTransformer +(63) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(63) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(64) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(67) Project +(68) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(69) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(72) Project +(73) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(77) Project +(78) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(79) Filter +(80) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(80) BroadcastExchange +(81) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(82) Project +(83) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(87) Project +(88) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(88) Scan parquet +(89) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(90) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(90) BroadcastExchange +(91) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(92) Project +(93) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(93) HashAggregate +(94) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(94) Exchange +(95) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) HashAggregate +(96) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(96) Exchange +(97) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(98) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(98) AdaptiveSparkPlan +(99) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/1.txt index 090a9522f13a..468ad7c5d3aa 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt index 808f58b58fa5..ce956550a45a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt @@ -1,64 +1,65 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- BroadcastQueryStage (8), Statistics(X) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ FilterExecTransformer (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- BroadcastQueryStage (18), Statistics(X) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ FilterExecTransformer (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- BroadcastQueryStage (27), Statistics(X) - +- ColumnarBroadcastExchange (26) - +- ^ FilterExecTransformer (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (44) + +- TakeOrderedAndProjectExecTransformer (43) + +- ^ ProjectExecTransformer (41) + +- ^ RegularHashAggregateExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- VeloxAppendBatches (35) + +- ^ ProjectExecTransformer (33) + +- ^ RegularHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ FilterExecTransformer (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ FilterExecTransformer (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ FilterExecTransformer (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- BroadcastHashJoin Inner BuildRight (61) - :- Project (57) - : +- BroadcastHashJoin Inner BuildRight (56) - : :- Project (51) - : : +- BroadcastHashJoin Inner BuildRight (50) - : : :- Filter (45) - : : : +- Scan parquet (44) - : : +- BroadcastExchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (55) - : +- Project (54) - : +- Filter (53) - : +- Scan parquet (52) - +- BroadcastExchange (60) - +- Filter (59) - +- Scan parquet (58) + TakeOrderedAndProject (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- BroadcastHashJoin Inner BuildRight (62) + :- Project (58) + : +- BroadcastHashJoin Inner BuildRight (57) + : :- Project (52) + : : +- BroadcastHashJoin Inner BuildRight (51) + : : :- Filter (46) + : : : +- Scan parquet (45) + : : +- BroadcastExchange (50) + : : +- Project (49) + : : +- Filter (48) + : : +- Scan parquet (47) + : +- BroadcastExchange (56) + : +- Project (55) + : +- Filter (54) + : +- Scan parquet (53) + +- BroadcastExchange (61) + +- Filter (60) + +- Scan parquet (59) (1) Scan parquet @@ -197,7 +198,7 @@ Join condition: None Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(32) FlushableHashAggregateExecTransformer +(32) RegularHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -212,158 +213,162 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(35) ColumnarExchange +(35) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(36) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(37) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(37) InputAdapter +(38) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(38) InputIteratorTransformer +(39) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(40) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(40) ProjectExecTransformer +(41) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(41) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(43) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(43) VeloxColumnarToRowExec +(44) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(44) Scan parquet +(45) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(46) Scan parquet +(47) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(48) Project +(49) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(51) Project +(52) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(52) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(53) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(54) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(55) BroadcastExchange +(56) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(57) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(58) Scan parquet +(59) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(59) Filter +(60) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(60) BroadcastExchange +(61) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(61) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(62) Project +(63) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(63) HashAggregate +(64) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(64) Exchange +(65) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(66) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(66) TakeOrderedAndProject +(67) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt index e6f8232f12c4..29763329fd30 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt @@ -1,55 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7), Statistics(X) - : +- ColumnarBroadcastExchange (6) - : +- ^ FilterExecTransformer (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17), Statistics(X) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FilterExecTransformer (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ FilterExecTransformer (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FilterExecTransformer (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (57) - +- Exchange (56) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- BroadcastHashJoin Inner BuildRight (50) - :- Project (45) - : +- BroadcastHashJoin Inner BuildRight (44) - : :- Filter (40) - : : +- Scan parquet (39) - : +- BroadcastExchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- BroadcastExchange (49) - +- Project (48) - +- Filter (47) - +- Scan parquet (46) + Sort (58) + +- Exchange (57) + +- Filter (56) + +- HashAggregate (55) + +- Exchange (54) + +- HashAggregate (53) + +- Project (52) + +- BroadcastHashJoin Inner BuildRight (51) + :- Project (46) + : +- BroadcastHashJoin Inner BuildRight (45) + : :- Filter (41) + : : +- Scan parquet (40) + : +- BroadcastExchange (44) + : +- Filter (43) + : +- Scan parquet (42) + +- BroadcastExchange (50) + +- Project (49) + +- Filter (48) + +- Scan parquet (47) (1) Scan parquet @@ -189,356 +190,360 @@ Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery su Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(34) InputAdapter +(35) InputAdapter Input [2]: [ps_partkey#X, value#X] -(35) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(36) SortExecTransformer +(37) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(38) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(39) Scan parquet +(40) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(41) Scan parquet +(42) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(42) Filter +(43) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(43) BroadcastExchange +(44) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(45) Project +(46) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(46) Scan parquet +(47) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(48) Project +(49) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(51) Project +(52) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(52) HashAggregate +(53) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(53) Exchange +(54) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(55) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(55) Filter +(56) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(56) Exchange +(57) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(58) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(58) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 30 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (99) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (82) - +- ^ ProjectExecTransformer (80) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) - :- ^ ProjectExecTransformer (66) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) - : :- ^ FilterExecTransformer (60) - : : +- ^ Scan parquet (59) - : +- ^ InputIteratorTransformer (64) - : +- BroadcastQueryStage (62), Statistics(X) - : +- ReusedExchange (61) - +- ^ InputIteratorTransformer (70) - +- BroadcastQueryStage (68), Statistics(X) - +- ReusedExchange (67) + VeloxColumnarToRowExec (83) + +- ^ ProjectExecTransformer (81) + +- ^ RegularHashAggregateExecTransformer (80) + +- ^ InputIteratorTransformer (79) + +- ShuffleQueryStage (77), Statistics(X) + +- ColumnarExchange (76) + +- ^ FlushableHashAggregateExecTransformer (74) + +- ^ ProjectExecTransformer (73) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (72) + :- ^ ProjectExecTransformer (67) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + : :- ^ FilterExecTransformer (61) + : : +- ^ Scan parquet (60) + : +- ^ InputIteratorTransformer (65) + : +- BroadcastQueryStage (63), Statistics(X) + : +- ReusedExchange (62) + +- ^ InputIteratorTransformer (71) + +- BroadcastQueryStage (69), Statistics(X) + +- ReusedExchange (68) +- == Initial Plan == - HashAggregate (98) - +- Exchange (97) - +- HashAggregate (96) - +- Project (95) - +- BroadcastHashJoin Inner BuildRight (94) - :- Project (89) - : +- BroadcastHashJoin Inner BuildRight (88) - : :- Filter (84) - : : +- Scan parquet (83) - : +- BroadcastExchange (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (93) - +- Project (92) - +- Filter (91) - +- Scan parquet (90) - - -(59) Scan parquet + HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Project (96) + +- BroadcastHashJoin Inner BuildRight (95) + :- Project (90) + : +- BroadcastHashJoin Inner BuildRight (89) + : :- Filter (85) + : : +- Scan parquet (84) + : +- BroadcastExchange (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (94) + +- Project (93) + +- Filter (92) + +- Scan parquet (91) + + +(60) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(60) FilterExecTransformer +(61) FilterExecTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: isnotnull(ps_suppkey#X) -(61) ReusedExchange [Reuses operator id: 6] +(62) ReusedExchange [Reuses operator id: 6] Output [2]: [s_suppkey#X, s_nationkey#X] -(62) BroadcastQueryStage +(63) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(63) InputAdapter +(64) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(64) InputIteratorTransformer +(65) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(65) BroadcastHashJoinExecTransformer +(66) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(66) ProjectExecTransformer +(67) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(67) ReusedExchange [Reuses operator id: 16] +(68) ReusedExchange [Reuses operator id: 16] Output [1]: [n_nationkey#X] -(68) BroadcastQueryStage +(69) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(69) InputAdapter +(70) InputAdapter Input [1]: [n_nationkey#X] -(70) InputIteratorTransformer +(71) InputIteratorTransformer Input [1]: [n_nationkey#X] -(71) BroadcastHashJoinExecTransformer +(72) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(72) ProjectExecTransformer +(73) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(73) FlushableHashAggregateExecTransformer +(74) FlushableHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(74) WholeStageCodegenTransformer (X) +(75) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(75) ColumnarExchange +(76) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(77) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(77) InputAdapter +(78) InputAdapter Input [2]: [sum#X, isEmpty#X] -(78) InputIteratorTransformer +(79) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(79) RegularHashAggregateExecTransformer +(80) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(80) ProjectExecTransformer +(81) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(81) WholeStageCodegenTransformer (X) +(82) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(82) VeloxColumnarToRowExec +(83) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(83) Scan parquet +(84) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(84) Filter +(85) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(85) Scan parquet +(86) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(87) BroadcastExchange +(88) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(89) Project +(90) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(90) Scan parquet +(91) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(91) Filter +(92) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(92) Project +(93) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(93) BroadcastExchange +(94) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(94) BroadcastHashJoin +(95) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(95) Project +(96) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(96) HashAggregate +(97) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(97) Exchange +(98) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(98) HashAggregate +(99) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(99) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt index 261e4fc821ea..8b6161e1f224 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt @@ -1,41 +1,42 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (43) +- == Final Plan == - VeloxColumnarToRowExec (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (29) + +- ^ SortExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- VeloxAppendBatches (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) + Sort (42) + +- Exchange (41) + +- HashAggregate (40) + +- Exchange (39) + +- HashAggregate (38) + +- Project (37) + +- BroadcastHashJoin Inner BuildLeft (36) + :- BroadcastExchange (32) + : +- Filter (31) + : +- Scan parquet (30) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -132,97 +133,101 @@ Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_ Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) SortExecTransformer +(27) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(27) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(30) Filter +(31) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(31) BroadcastExchange +(32) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(32) Scan parquet +(33) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(33) Filter +(34) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(34) Project +(35) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(35) BroadcastHashJoin +(36) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(36) Project +(37) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(37) HashAggregate +(38) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(38) Exchange +(39) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(40) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(40) Exchange +(41) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(42) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(43) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt index 7ccf5dafe28e..f84ec67ca125 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt @@ -1,49 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FilterExecTransformer (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ FilterExecTransformer (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- BroadcastHashJoin LeftOuter BuildRight (42) - :- Scan parquet (37) - +- BroadcastExchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- BroadcastHashJoin LeftOuter BuildRight (43) + :- Scan parquet (38) + +- BroadcastExchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -175,110 +176,114 @@ Results [2]: [c_count#X, count(1)#X AS custdist#X] Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(33) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(35) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(38) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(43) Project +(44) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) Exchange +(46) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(48) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(49) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(50) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(51) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(52) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt index 8655ec75dd6f..ef33284c5ef7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt @@ -1,35 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- BroadcastHashJoin Inner BuildRight (29) - :- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- BroadcastExchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- BroadcastHashJoin Inner BuildRight (30) + :- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- BroadcastExchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -86,7 +87,7 @@ Join condition: None Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] @@ -97,96 +98,100 @@ Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(23) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(25) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) Scan parquet +(27) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(28) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) BroadcastExchange +(29) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastHashJoin +(30) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(30) Project +(31) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(32) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) Exchange +(33) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] -(34) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt index 0853cd6948f9..978722c255d7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt @@ -1,42 +1,43 @@ == Physical Plan == -AdaptiveSparkPlan (41) +AdaptiveSparkPlan (42) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- AQEShuffleRead (25) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (27) + +- AQEShuffleRead (26) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (40) - +- Exchange (39) - +- Project (38) - +- BroadcastHashJoin Inner BuildLeft (37) - :- BroadcastExchange (29) - : +- Filter (28) - : +- Scan parquet (27) - +- Filter (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- Filter (31) - +- Scan parquet (30) + Sort (41) + +- Exchange (40) + +- Project (39) + +- BroadcastHashJoin Inner BuildLeft (38) + :- BroadcastExchange (30) + : +- Filter (29) + : +- Scan parquet (28) + +- Filter (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- Filter (32) + +- Scan parquet (31) (1) Scan parquet @@ -137,247 +138,251 @@ Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_ Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(25) AQEShuffleRead +(26) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(26) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) Scan parquet +(28) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(28) Filter +(29) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) Scan parquet +(31) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(32) Project +(33) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(33) HashAggregate +(34) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(34) Exchange +(35) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(36) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(36) Filter +(37) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(37) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(38) Project +(39) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(39) Exchange +(40) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) Sort +(41) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(41) AdaptiveSparkPlan +(42) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 19 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (66) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (57) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ ProjectExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ FilterExecTransformer (43) - +- ^ Scan parquet (42) + VeloxColumnarToRowExec (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- ^ ProjectExecTransformer (47) + +- ^ FlushableHashAggregateExecTransformer (46) + +- ^ ProjectExecTransformer (45) + +- ^ FilterExecTransformer (44) + +- ^ Scan parquet (43) +- == Initial Plan == - HashAggregate (65) - +- HashAggregate (64) - +- HashAggregate (63) - +- Exchange (62) - +- HashAggregate (61) - +- Project (60) - +- Filter (59) - +- Scan parquet (58) + HashAggregate (66) + +- HashAggregate (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- Project (61) + +- Filter (60) + +- Scan parquet (59) -(42) Scan parquet +(43) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(43) FilterExecTransformer +(44) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(45) FlushableHashAggregateExecTransformer +(46) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(46) ProjectExecTransformer +(47) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(49) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(53) ProjectExecTransformer +(54) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(54) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(55) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(56) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(57) VeloxColumnarToRowExec +(58) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(58) Scan parquet +(59) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(59) Filter +(60) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(60) Project +(61) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(61) HashAggregate +(62) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(62) Exchange +(63) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(64) HashAggregate +(65) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(65) HashAggregate +(66) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(66) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt index 78c65e72e6c8..fe411808b790 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt @@ -1,53 +1,54 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (57) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ FilterExecTransformer (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ FilterExecTransformer (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- BroadcastHashJoin Inner BuildRight (46) - :- BroadcastHashJoin LeftAnti BuildRight (42) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (45) - +- Filter (44) - +- Scan parquet (43) + Sort (56) + +- Exchange (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- BroadcastHashJoin Inner BuildRight (47) + :- BroadcastHashJoin LeftAnti BuildRight (43) + : :- Filter (38) + : : +- Scan parquet (37) + : +- BroadcastExchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- BroadcastExchange (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -176,136 +177,140 @@ Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_c Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) ColumnarExchange +(29) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(30) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(36) Scan parquet +(37) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(37) Filter +(38) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(38) Scan parquet +(39) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(39) Filter +(40) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(40) Project +(41) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: LeftAnti Join condition: None -(43) Scan parquet +(44) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(45) BroadcastExchange +(46) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(47) Project +(48) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(48) HashAggregate +(49) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(49) Exchange +(50) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(51) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(51) HashAggregate +(52) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(52) Exchange +(53) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(54) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(56) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(56) AdaptiveSparkPlan +(57) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/17.txt index 62612096db42..e499293269fc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/17.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (14) - +- ^ ProjectExecTransformer (12) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- RowToVeloxColumnar (2) - +- LocalTableScan (1) + VeloxColumnarToRowExec (15) + +- ^ ProjectExecTransformer (13) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ RegularHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == - HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin Inner BuildRight (30) - :- Project (22) - : +- BroadcastHashJoin Inner BuildRight (21) - : :- Filter (16) - : : +- Scan parquet (15) - : +- BroadcastExchange (20) - : +- Project (19) - : +- Filter (18) - : +- Scan parquet (17) - +- BroadcastExchange (29) - +- Filter (28) - +- HashAggregate (27) - +- Exchange (26) - +- HashAggregate (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin Inner BuildRight (31) + :- Project (23) + : +- BroadcastHashJoin Inner BuildRight (22) + : :- Filter (17) + : : +- Scan parquet (16) + : +- BroadcastExchange (21) + : +- Project (20) + : +- Filter (19) + : +- Scan parquet (18) + +- BroadcastExchange (30) + +- Filter (29) + +- HashAggregate (28) + +- Exchange (27) + +- HashAggregate (26) + +- Filter (25) + +- Scan parquet (24) (1) LocalTableScan @@ -47,7 +48,7 @@ Input [1]: [l_extendedprice#X] (4) InputIteratorTransformer Input [1]: [l_extendedprice#X] -(5) FlushableHashAggregateExecTransformer +(5) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] @@ -58,143 +59,147 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(8) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [2]: [sum#X, isEmpty#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(14) VeloxColumnarToRowExec +(15) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(15) Scan parquet +(16) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(16) Filter +(17) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(17) Scan parquet +(18) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(18) Filter +(19) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(19) Project +(20) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(20) BroadcastExchange +(21) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastHashJoin +(22) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(22) Project +(23) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(23) Scan parquet +(24) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(25) HashAggregate +(26) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(26) Exchange +(27) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) HashAggregate +(28) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(28) Filter +(29) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(31) Project +(32) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(32) HashAggregate +(33) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] -(35) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt index 58751c9b885f..8899b0c255ae 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt @@ -1,80 +1,82 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- TakeOrderedAndProjectExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ FilterExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23), Statistics(X) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- BroadcastQueryStage (32), Statistics(X) - +- ReusedExchange (31) + VeloxColumnarToRowExec (55) + +- TakeOrderedAndProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- VeloxAppendBatches (47) + +- ^ ProjectExecTransformer (45) + +- ^ RegularHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) + :- ^ ProjectExecTransformer (29) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (28) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (27) + : :- ^ FilterExecTransformer (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (26) + : +- BroadcastQueryStage (24), Statistics(X) + : +- ColumnarBroadcastExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FilterExecTransformer (20) + : +- ^ RegularHashAggregateExecTransformer (19) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ RegularHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (41) + +- BroadcastQueryStage (39), Statistics(X) + +- ColumnarBroadcastExchange (38) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (36) + :- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (35) + +- BroadcastQueryStage (33), Statistics(X) + +- ReusedExchange (32) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (68) - : +- BroadcastHashJoin Inner BuildLeft (67) - : :- BroadcastExchange (56) - : : +- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastHashJoin LeftSemi BuildRight (66) - : :- Filter (58) - : : +- Scan parquet (57) - : +- BroadcastExchange (65) - : +- Project (64) - : +- Filter (63) - : +- HashAggregate (62) - : +- Exchange (61) - : +- HashAggregate (60) - : +- Scan parquet (59) - +- BroadcastExchange (79) - +- BroadcastHashJoin LeftSemi BuildRight (78) - :- Filter (70) - : +- Scan parquet (69) - +- BroadcastExchange (77) - +- Project (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Scan parquet (71) + TakeOrderedAndProject (87) + +- HashAggregate (86) + +- Exchange (85) + +- HashAggregate (84) + +- Project (83) + +- BroadcastHashJoin Inner BuildRight (82) + :- Project (70) + : +- BroadcastHashJoin Inner BuildLeft (69) + : :- BroadcastExchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- BroadcastHashJoin LeftSemi BuildRight (68) + : :- Filter (60) + : : +- Scan parquet (59) + : +- BroadcastExchange (67) + : +- Project (66) + : +- Filter (65) + : +- HashAggregate (64) + : +- Exchange (63) + : +- HashAggregate (62) + : +- Scan parquet (61) + +- BroadcastExchange (81) + +- BroadcastHashJoin LeftSemi BuildRight (80) + :- Filter (72) + : +- Scan parquet (71) + +- BroadcastExchange (79) + +- Project (78) + +- Filter (77) + +- HashAggregate (76) + +- Exchange (75) + +- HashAggregate (74) + +- Scan parquet (73) (1) Scan parquet @@ -123,7 +125,7 @@ Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) FlushableHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] @@ -138,341 +140,349 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [1]: [l_orderkey#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [1]: [l_orderkey#X] -(26) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(27) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(30) FilterExecTransformer +(31) FilterExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Arguments: isnotnull(l_orderkey#X) -(31) ReusedExchange [Reuses operator id: 22] +(32) ReusedExchange [Reuses operator id: 23] Output [1]: [l_orderkey#X] -(32) BroadcastQueryStage +(33) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [1]: [l_orderkey#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [l_orderkey#X] -(35) BroadcastHashJoinExecTransformer +(36) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(37) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(39) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(41) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(43) FlushableHashAggregateExecTransformer +(44) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(47) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: X + +(48) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(49) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(50) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(51) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(51) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(52) TakeOrderedAndProjectExecTransformer +(54) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(53) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(54) Scan parquet +(56) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(56) BroadcastExchange +(58) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(59) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(58) Filter +(60) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(59) Scan parquet +(61) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(60) HashAggregate +(62) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(61) Exchange +(63) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(63) Filter +(65) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(64) Project +(66) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(65) BroadcastExchange +(67) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(67) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(68) Project +(70) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(69) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(71) Scan parquet +(73) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(74) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(75) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(76) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(77) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(78) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) BroadcastExchange +(79) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(79) BroadcastExchange +(81) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(81) Project +(83) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(82) HashAggregate +(84) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(83) Exchange +(85) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(86) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(85) TakeOrderedAndProject +(87) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(86) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt index e2f0c051b69a..0373360159eb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt @@ -1,34 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (32) - +- Exchange (31) - +- HashAggregate (30) - +- Project (29) - +- BroadcastHashJoin Inner BuildRight (28) - :- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- BroadcastExchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -85,7 +86,7 @@ Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -96,92 +97,96 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(20) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(22) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(23) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(24) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(25) Scan parquet +(26) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(27) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) BroadcastExchange +(28) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(30) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(31) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) Exchange +(32) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(32) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(33) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt index 3b7e5334a3ba..8a1415a5bca2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt @@ -1,90 +1,91 @@ == Physical Plan == -AdaptiveSparkPlan (96) +AdaptiveSparkPlan (97) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- AQEShuffleRead (59) - +- ShuffleQueryStage (58), Statistics(X) - +- ColumnarExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (54) - :- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (43) - : +- BroadcastQueryStage (41), Statistics(X) - : +- ColumnarBroadcastExchange (40) - : +- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (37) - : :- ^ InputIteratorTransformer (18) - : : +- BroadcastQueryStage (16), Statistics(X) - : : +- ColumnarBroadcastExchange (15) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (13) - : : :- ^ FilterExecTransformer (4) - : : : +- ^ Scan parquet (3) - : : +- ^ InputIteratorTransformer (12) - : : +- BroadcastQueryStage (10), Statistics(X) - : : +- ColumnarBroadcastExchange (9) - : : +- ^ ProjectExecTransformer (7) - : : +- ^ FilterExecTransformer (6) - : : +- ^ Scan parquet (5) - : +- ^ FilterExecTransformer (36) - : +- ^ ProjectExecTransformer (35) - : +- ^ RegularHashAggregateExecTransformer (34) - : +- ^ InputIteratorTransformer (33) - : +- ShuffleQueryStage (31), Statistics(X) - : +- ColumnarExchange (30) - : +- ^ ProjectExecTransformer (28) - : +- ^ FlushableHashAggregateExecTransformer (27) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ ProjectExecTransformer (21) - : : +- ^ FilterExecTransformer (20) - : : +- ^ Scan parquet (19) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23), Statistics(X) - : +- ReusedExchange (22) - +- ^ InputIteratorTransformer (53) - +- BroadcastQueryStage (51), Statistics(X) - +- ColumnarBroadcastExchange (50) - +- ^ ProjectExecTransformer (48) - +- ^ FilterExecTransformer (47) - +- ^ Scan parquet (46) + VeloxColumnarToRowExec (61) + +- AQEShuffleRead (60) + +- ShuffleQueryStage (59), Statistics(X) + +- ColumnarExchange (58) + +- VeloxAppendBatches (57) + +- ^ ProjectExecTransformer (55) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (54) + :- ^ ProjectExecTransformer (45) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (44) + : :- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (43) + : +- BroadcastQueryStage (41), Statistics(X) + : +- ColumnarBroadcastExchange (40) + : +- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (37) + : :- ^ InputIteratorTransformer (18) + : : +- BroadcastQueryStage (16), Statistics(X) + : : +- ColumnarBroadcastExchange (15) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (13) + : : :- ^ FilterExecTransformer (4) + : : : +- ^ Scan parquet (3) + : : +- ^ InputIteratorTransformer (12) + : : +- BroadcastQueryStage (10), Statistics(X) + : : +- ColumnarBroadcastExchange (9) + : : +- ^ ProjectExecTransformer (7) + : : +- ^ FilterExecTransformer (6) + : : +- ^ Scan parquet (5) + : +- ^ FilterExecTransformer (36) + : +- ^ ProjectExecTransformer (35) + : +- ^ RegularHashAggregateExecTransformer (34) + : +- ^ InputIteratorTransformer (33) + : +- ShuffleQueryStage (31), Statistics(X) + : +- ColumnarExchange (30) + : +- ^ ProjectExecTransformer (28) + : +- ^ FlushableHashAggregateExecTransformer (27) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) + : :- ^ ProjectExecTransformer (21) + : : +- ^ FilterExecTransformer (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (25) + : +- BroadcastQueryStage (23), Statistics(X) + : +- ReusedExchange (22) + +- ^ InputIteratorTransformer (53) + +- BroadcastQueryStage (51), Statistics(X) + +- ColumnarBroadcastExchange (50) + +- ^ ProjectExecTransformer (48) + +- ^ FilterExecTransformer (47) + +- ^ Scan parquet (46) +- == Initial Plan == - Sort (95) - +- Exchange (94) - +- Project (93) - +- BroadcastHashJoin Inner BuildRight (92) - :- Project (87) - : +- BroadcastHashJoin LeftSemi BuildRight (86) - : :- Filter (62) - : : +- Scan parquet (61) - : +- BroadcastExchange (85) - : +- Project (84) - : +- BroadcastHashJoin Inner BuildLeft (83) - : :- BroadcastExchange (70) - : : +- BroadcastHashJoin LeftSemi BuildRight (69) - : : :- Filter (64) - : : : +- Scan parquet (63) - : : +- BroadcastExchange (68) - : : +- Project (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Filter (82) - : +- HashAggregate (81) - : +- Exchange (80) - : +- HashAggregate (79) - : +- BroadcastHashJoin LeftSemi BuildRight (78) - : :- Project (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- BroadcastExchange (77) - : +- Project (76) - : +- Filter (75) - : +- Scan parquet (74) - +- BroadcastExchange (91) - +- Project (90) - +- Filter (89) - +- Scan parquet (88) + Sort (96) + +- Exchange (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (88) + : +- BroadcastHashJoin LeftSemi BuildRight (87) + : :- Filter (63) + : : +- Scan parquet (62) + : +- BroadcastExchange (86) + : +- Project (85) + : +- BroadcastHashJoin Inner BuildLeft (84) + : :- BroadcastExchange (71) + : : +- BroadcastHashJoin LeftSemi BuildRight (70) + : : :- Filter (65) + : : : +- Scan parquet (64) + : : +- BroadcastExchange (69) + : : +- Project (68) + : : +- Filter (67) + : : +- Scan parquet (66) + : +- Filter (83) + : +- HashAggregate (82) + : +- Exchange (81) + : +- HashAggregate (80) + : +- BroadcastHashJoin LeftSemi BuildRight (79) + : :- Project (74) + : : +- Filter (73) + : : +- Scan parquet (72) + : +- BroadcastExchange (78) + : +- Project (77) + : +- Filter (76) + : +- Scan parquet (75) + +- BroadcastExchange (92) + +- Project (91) + +- Filter (90) + +- Scan parquet (89) (1) Scan parquet @@ -329,195 +330,199 @@ Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] Input [2]: [s_name#X, s_address#X] Arguments: false -(57) ColumnarExchange +(57) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(58) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(58) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(59) AQEShuffleRead +(60) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(60) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(61) Scan parquet +(62) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(63) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(63) Scan parquet +(64) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(64) Filter +(65) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(65) Scan parquet +(66) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(66) Filter +(67) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(67) Project +(68) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(69) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) BroadcastHashJoin +(70) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(70) BroadcastExchange +(71) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(71) Scan parquet +(72) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(72) Filter +(73) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(73) Project +(74) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(74) Scan parquet +(75) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(75) Filter +(76) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(76) Project +(77) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(77) BroadcastExchange +(78) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(79) HashAggregate +(80) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(80) Exchange +(81) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) HashAggregate +(82) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(82) Filter +(83) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(83) BroadcastHashJoin +(84) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(84) Project +(85) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(85) BroadcastExchange +(86) BroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(87) Project +(88) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(88) Scan parquet +(89) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(90) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(90) Project +(91) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(91) BroadcastExchange +(92) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(92) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(93) Project +(94) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(94) Exchange +(95) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Sort +(96) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(96) AdaptiveSparkPlan +(97) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt index f81832ea4c1d..48fc651f66f1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt @@ -1,84 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (91) +AdaptiveSparkPlan (92) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- BroadcastQueryStage (5), Statistics(X) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ FilterExecTransformer (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5), Statistics(X) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ FilterExecTransformer (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ FilterExecTransformer (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (90) - +- HashAggregate (89) - +- Exchange (88) - +- HashAggregate (87) - +- Project (86) - +- BroadcastHashJoin Inner BuildRight (85) - :- Project (80) - : +- BroadcastHashJoin Inner BuildRight (79) - : :- Project (74) - : : +- BroadcastHashJoin Inner BuildLeft (73) - : : :- BroadcastExchange (61) - : : : +- Filter (60) - : : : +- Scan parquet (59) - : : +- BroadcastHashJoin LeftAnti BuildRight (72) - : : :- BroadcastHashJoin LeftSemi BuildRight (67) - : : : :- Project (64) - : : : : +- Filter (63) - : : : : +- Scan parquet (62) - : : : +- BroadcastExchange (66) - : : : +- Scan parquet (65) - : : +- BroadcastExchange (71) - : : +- Project (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- BroadcastExchange (78) - : +- Project (77) - : +- Filter (76) - : +- Scan parquet (75) - +- BroadcastExchange (84) - +- Project (83) - +- Filter (82) - +- Scan parquet (81) + TakeOrderedAndProject (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (81) + : +- BroadcastHashJoin Inner BuildRight (80) + : :- Project (75) + : : +- BroadcastHashJoin Inner BuildLeft (74) + : : :- BroadcastExchange (62) + : : : +- Filter (61) + : : : +- Scan parquet (60) + : : +- BroadcastHashJoin LeftAnti BuildRight (73) + : : :- BroadcastHashJoin LeftSemi BuildRight (68) + : : : :- Project (65) + : : : : +- Filter (64) + : : : : +- Scan parquet (63) + : : : +- BroadcastExchange (67) + : : : +- Scan parquet (66) + : : +- BroadcastExchange (72) + : : +- Project (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- BroadcastExchange (79) + : +- Project (78) + : +- Filter (77) + : +- Scan parquet (76) + +- BroadcastExchange (85) + +- Project (84) + +- Filter (83) + +- Scan parquet (82) (1) Scan parquet @@ -290,7 +291,7 @@ Join condition: None Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(49) FlushableHashAggregateExecTransformer +(49) RegularHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] @@ -305,195 +306,199 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(53) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [2]: [s_name#X, count#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(58) VeloxColumnarToRowExec +(59) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(59) Scan parquet +(60) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(60) Filter +(61) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(61) BroadcastExchange +(62) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(62) Scan parquet +(63) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(63) Filter +(64) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(64) Project +(65) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(66) BroadcastExchange +(67) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(67) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(68) Scan parquet +(69) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(69) Filter +(70) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(70) Project +(71) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(71) BroadcastExchange +(72) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(74) Project +(75) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(75) Scan parquet +(76) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(76) Filter +(77) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(77) Project +(78) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(78) BroadcastExchange +(79) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(79) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(80) Project +(81) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(81) Scan parquet +(82) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(82) Filter +(83) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(83) Project +(84) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(84) BroadcastExchange +(85) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(85) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(86) Project +(87) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(87) HashAggregate +(88) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(88) Exchange +(89) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(90) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(90) TakeOrderedAndProject +(91) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(91) AdaptiveSparkPlan +(92) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt index 5dac875505ea..b740990ca780 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (39) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (27) + +- ^ SortExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- VeloxAppendBatches (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (37) - +- Exchange (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- BroadcastHashJoin LeftAnti BuildRight (31) - :- Filter (28) - : +- Scan parquet (27) - +- BroadcastExchange (30) - +- Scan parquet (29) + Sort (38) + +- Exchange (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- BroadcastHashJoin LeftAnti BuildRight (32) + :- Filter (29) + : +- Scan parquet (28) + +- BroadcastExchange (31) + +- Scan parquet (30) (1) Scan parquet @@ -119,222 +120,226 @@ Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacc Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) ColumnarExchange +(20) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(21) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(22) InputAdapter +(23) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) SortExecTransformer +(25) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(27) Scan parquet +(28) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(28) Filter +(29) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(29) Scan parquet +(30) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(30) BroadcastExchange +(31) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(31) BroadcastHashJoin +(32) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(32) Project +(33) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(33) HashAggregate +(34) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(34) Exchange +(35) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(36) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(36) Exchange +(37) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Sort +(38) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(38) AdaptiveSparkPlan +(39) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (58) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ RegularHashAggregateExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (51) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ InputIteratorTransformer (48) + +- ShuffleQueryStage (46), Statistics(X) + +- ColumnarExchange (45) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) + HashAggregate (57) + +- Exchange (56) + +- HashAggregate (55) + +- Project (54) + +- Filter (53) + +- Scan parquet (52) -(39) Scan parquet +(40) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(40) FilterExecTransformer +(41) FilterExecTransformer Input [2]: [c_phone#X, c_acctbal#X] Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(41) ProjectExecTransformer +(42) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(42) FlushableHashAggregateExecTransformer +(43) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(43) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(44) ColumnarExchange +(45) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(46) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(46) InputAdapter +(47) InputAdapter Input [2]: [sum#X, count#X] -(47) InputIteratorTransformer +(48) InputIteratorTransformer Input [2]: [sum#X, count#X] -(48) RegularHashAggregateExecTransformer +(49) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(49) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(50) VeloxColumnarToRowExec +(51) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(51) Scan parquet +(52) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(52) Filter +(53) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(53) Project +(54) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(54) HashAggregate +(55) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(55) Exchange +(56) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(57) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) AdaptiveSparkPlan +(58) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (58) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ RegularHashAggregateExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (51) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ InputIteratorTransformer (48) + +- ShuffleQueryStage (46), Statistics(X) + +- ColumnarExchange (45) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) \ No newline at end of file + HashAggregate (57) + +- Exchange (56) + +- HashAggregate (55) + +- Project (54) + +- Filter (53) + +- Scan parquet (52) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt index 7ee3b1a0cfd4..5eb8568582af 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- TakeOrderedAndProjectExecTransformer (33) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- BroadcastQueryStage (18), Statistics(X) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FilterExecTransformer (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (35) + +- TakeOrderedAndProjectExecTransformer (34) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ RegularHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FilterExecTransformer (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildLeft (41) - : :- BroadcastExchange (38) - : : +- Project (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + TakeOrderedAndProject (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- BroadcastHashJoin Inner BuildRight (48) + :- Project (43) + : +- BroadcastHashJoin Inner BuildLeft (42) + : :- BroadcastExchange (39) + : : +- Project (38) + : : +- Filter (37) + : : +- Scan parquet (36) + : +- Filter (41) + : +- Scan parquet (40) + +- BroadcastExchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -146,7 +147,7 @@ Join condition: None Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) FlushableHashAggregateExecTransformer +(23) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -161,133 +162,137 @@ Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: X + +(27) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(33) TakeOrderedAndProjectExecTransformer +(34) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(34) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(35) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(36) Filter +(37) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(37) Project +(38) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(38) BroadcastExchange +(39) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) Scan parquet +(40) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(42) Project +(43) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(43) Scan parquet +(44) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(45) Project +(46) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) BroadcastExchange +(47) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(48) Project +(49) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(49) HashAggregate +(50) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(50) Exchange +(51) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(52) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(52) TakeOrderedAndProject +(53) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt index 65a268b41109..7c123f45309b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt @@ -1,43 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9), Statistics(X) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- BroadcastHashJoin LeftSemi BuildRight (37) - :- Project (32) - : +- Filter (31) - : +- Scan parquet (30) - +- BroadcastExchange (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (44) + +- Exchange (43) + +- HashAggregate (42) + +- Exchange (41) + +- HashAggregate (40) + +- Project (39) + +- BroadcastHashJoin LeftSemi BuildRight (38) + :- Project (33) + : +- Filter (32) + : +- Scan parquet (31) + +- BroadcastExchange (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -138,101 +139,105 @@ Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(27) SortExecTransformer +(28) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(30) Scan parquet +(31) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(31) Filter +(32) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(32) Project +(33) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(33) Scan parquet +(34) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(34) Filter +(35) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(35) Project +(36) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(36) BroadcastExchange +(37) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(37) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(38) Project +(39) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(39) HashAggregate +(40) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(40) Exchange +(41) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) HashAggregate +(42) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(42) Exchange +(43) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(44) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt index a86ee299c7c3..d2b7ce10fb33 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt @@ -1,91 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (100) +AdaptiveSparkPlan (101) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45), Statistics(X) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61), Statistics(X) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (99) - +- Exchange (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Project (71) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (92) - +- Project (91) - +- Filter (90) - +- Scan parquet (89) + Sort (100) + +- Exchange (99) + +- HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- BroadcastHashJoin Inner BuildRight (94) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Project (72) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (93) + +- Project (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -342,201 +343,205 @@ Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS reven Input [2]: [n_name#X, revenue#X] Arguments: false -(59) ColumnarExchange +(59) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(60) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(61) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(61) InputAdapter +(62) InputAdapter Input [2]: [n_name#X, revenue#X] -(62) InputIteratorTransformer +(63) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(63) SortExecTransformer +(64) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(65) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(67) Filter +(68) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(68) BroadcastExchange +(69) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(71) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(73) Project +(74) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(75) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(78) Project +(79) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(79) Scan parquet +(80) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(81) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(81) BroadcastExchange +(82) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(83) Project +(84) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(84) Scan parquet +(85) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(86) BroadcastExchange +(87) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(88) Project +(89) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(89) Scan parquet +(90) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(91) Project +(92) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(92) BroadcastExchange +(93) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(93) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(94) Project +(95) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(95) HashAggregate +(96) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(96) Exchange +(97) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(98) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(98) Exchange +(99) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Sort +(100) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(100) AdaptiveSparkPlan +(101) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/6.txt index 12d6c3ea85e4..415d85f46ee5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnot Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * l_discount#X) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt index 571b6c13b6fd..311144d09ec9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt @@ -1,85 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ SortExecTransformer (58) - +- ^ InputIteratorTransformer (57) - +- ShuffleQueryStage (55), Statistics(X) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16), Statistics(X) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ FilterExecTransformer (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25), Statistics(X) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ FilterExecTransformer (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ FilterExecTransformer (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40), Statistics(X) - +- ReusedExchange (39) + VeloxColumnarToRowExec (61) + +- ^ SortExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- VeloxAppendBatches (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ FilterExecTransformer (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ FilterExecTransformer (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == - Sort (92) - +- Exchange (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (63) - : : : : : +- Filter (62) - : : : : : +- Scan parquet (61) - : : : : +- Filter (65) - : : : : +- Scan parquet (64) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (85) - +- Filter (84) - +- Scan parquet (83) + Sort (93) + +- Exchange (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- BroadcastHashJoin Inner BuildRight (87) + :- Project (83) + : +- BroadcastHashJoin Inner BuildRight (82) + : :- Project (78) + : : +- BroadcastHashJoin Inner BuildRight (77) + : : :- Project (73) + : : : +- BroadcastHashJoin Inner BuildRight (72) + : : : :- Project (68) + : : : : +- BroadcastHashJoin Inner BuildLeft (67) + : : : : :- BroadcastExchange (64) + : : : : : +- Filter (63) + : : : : : +- Scan parquet (62) + : : : : +- Filter (66) + : : : : +- Scan parquet (65) + : : : +- BroadcastExchange (71) + : : : +- Filter (70) + : : : +- Scan parquet (69) + : : +- BroadcastExchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- BroadcastExchange (81) + : +- Filter (80) + : +- Scan parquet (79) + +- BroadcastExchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -312,193 +313,197 @@ Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) ColumnarExchange +(54) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(55) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(56) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(56) InputAdapter +(57) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(57) InputIteratorTransformer +(58) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) SortExecTransformer +(59) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(59) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(60) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(61) Scan parquet +(62) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(63) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(63) BroadcastExchange +(64) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(64) Scan parquet +(65) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(65) Filter +(66) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(66) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(67) Project +(68) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(68) Scan parquet +(69) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(69) Filter +(70) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(70) BroadcastExchange +(71) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(72) Project +(73) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(77) Project +(78) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(79) Filter +(80) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(80) BroadcastExchange +(81) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(82) Project +(83) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(87) Project +(88) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(88) HashAggregate +(89) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(89) Exchange +(90) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(91) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(91) Exchange +(92) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(93) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(93) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt index af98ad739c21..bcdebad75e77 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt @@ -1,116 +1,117 @@ == Physical Plan == -AdaptiveSparkPlan (129) +AdaptiveSparkPlan (130) +- == Final Plan == - VeloxColumnarToRowExec (84) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ FilterExecTransformer (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26), Statistics(X) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ FilterExecTransformer (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35), Statistics(X) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ FilterExecTransformer (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44), Statistics(X) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ FilterExecTransformer (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53), Statistics(X) - : +- ColumnarBroadcastExchange (52) - : +- ^ FilterExecTransformer (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63), Statistics(X) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ FilterExecTransformer (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (85) + +- ^ SortExecTransformer (83) + +- ^ InputIteratorTransformer (82) + +- ShuffleQueryStage (80), Statistics(X) + +- ColumnarExchange (79) + +- VeloxAppendBatches (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ShuffleQueryStage (72), Statistics(X) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ FilterExecTransformer (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ FilterExecTransformer (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ FilterExecTransformer (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ FilterExecTransformer (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ FilterExecTransformer (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ FilterExecTransformer (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ FilterExecTransformer (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (128) - +- Exchange (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- BroadcastHashJoin Inner BuildRight (122) - :- Project (117) - : +- BroadcastHashJoin Inner BuildRight (116) - : :- Project (112) - : : +- BroadcastHashJoin Inner BuildRight (111) - : : :- Project (107) - : : : +- BroadcastHashJoin Inner BuildRight (106) - : : : :- Project (102) - : : : : +- BroadcastHashJoin Inner BuildRight (101) - : : : : :- Project (97) - : : : : : +- BroadcastHashJoin Inner BuildRight (96) - : : : : : :- Project (92) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) - : : : : : : :- BroadcastExchange (88) - : : : : : : : +- Project (87) - : : : : : : : +- Filter (86) - : : : : : : : +- Scan parquet (85) - : : : : : : +- Filter (90) - : : : : : : +- Scan parquet (89) - : : : : : +- BroadcastExchange (95) - : : : : : +- Filter (94) - : : : : : +- Scan parquet (93) - : : : : +- BroadcastExchange (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- BroadcastExchange (105) - : : : +- Filter (104) - : : : +- Scan parquet (103) - : : +- BroadcastExchange (110) - : : +- Filter (109) - : : +- Scan parquet (108) - : +- BroadcastExchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- BroadcastExchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (129) + +- Exchange (128) + +- HashAggregate (127) + +- Exchange (126) + +- HashAggregate (125) + +- Project (124) + +- BroadcastHashJoin Inner BuildRight (123) + :- Project (118) + : +- BroadcastHashJoin Inner BuildRight (117) + : :- Project (113) + : : +- BroadcastHashJoin Inner BuildRight (112) + : : :- Project (108) + : : : +- BroadcastHashJoin Inner BuildRight (107) + : : : :- Project (103) + : : : : +- BroadcastHashJoin Inner BuildRight (102) + : : : : :- Project (98) + : : : : : +- BroadcastHashJoin Inner BuildRight (97) + : : : : : :- Project (93) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (92) + : : : : : : :- BroadcastExchange (89) + : : : : : : : +- Project (88) + : : : : : : : +- Filter (87) + : : : : : : : +- Scan parquet (86) + : : : : : : +- Filter (91) + : : : : : : +- Scan parquet (90) + : : : : : +- BroadcastExchange (96) + : : : : : +- Filter (95) + : : : : : +- Scan parquet (94) + : : : : +- BroadcastExchange (101) + : : : : +- Filter (100) + : : : : +- Scan parquet (99) + : : : +- BroadcastExchange (106) + : : : +- Filter (105) + : : : +- Scan parquet (104) + : : +- BroadcastExchange (111) + : : +- Filter (110) + : : +- Scan parquet (109) + : +- BroadcastExchange (116) + : +- Filter (115) + : +- Scan parquet (114) + +- BroadcastExchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) (1) Scan parquet @@ -449,251 +450,255 @@ Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.000 Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) ColumnarExchange +(78) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(79) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(80) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(80) InputAdapter +(81) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(81) InputIteratorTransformer +(82) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(82) SortExecTransformer +(83) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(83) WholeStageCodegenTransformer (X) +(84) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(84) VeloxColumnarToRowExec +(85) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(85) Scan parquet +(86) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(87) Project +(88) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(88) BroadcastExchange +(89) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) Scan parquet +(90) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(92) Project +(93) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(93) Scan parquet +(94) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(94) Filter +(95) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(95) BroadcastExchange +(96) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(96) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(97) Project +(98) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(98) Scan parquet +(99) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(99) Filter +(100) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(100) BroadcastExchange +(101) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(101) BroadcastHashJoin +(102) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(102) Project +(103) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(103) Scan parquet +(104) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(104) Filter +(105) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(105) BroadcastExchange +(106) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(106) BroadcastHashJoin +(107) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(107) Project +(108) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(108) Scan parquet +(109) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(109) Filter +(110) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(110) BroadcastExchange +(111) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(111) BroadcastHashJoin +(112) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(112) Project +(113) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(113) Scan parquet +(114) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(114) Filter +(115) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(115) BroadcastExchange +(116) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(116) BroadcastHashJoin +(117) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(117) Project +(118) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(118) Scan parquet +(119) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(119) Filter +(120) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(120) Project +(121) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(121) BroadcastExchange +(122) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(122) BroadcastHashJoin +(123) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(123) Project +(124) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(124) HashAggregate +(125) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) Exchange +(126) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(127) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(127) Exchange +(128) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(129) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(129) AdaptiveSparkPlan +(130) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt index c88eba3bb12e..4a0f66b2e746 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt @@ -1,89 +1,90 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (99) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- ^ SortExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (65) + +- ^ SortExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (97) - +- Exchange (96) - +- HashAggregate (95) - +- Exchange (94) - +- HashAggregate (93) - +- Project (92) - +- BroadcastHashJoin Inner BuildRight (91) - :- Project (87) - : +- BroadcastHashJoin Inner BuildRight (86) - : :- Project (82) - : : +- BroadcastHashJoin Inner BuildRight (81) - : : :- Project (77) - : : : +- BroadcastHashJoin Inner BuildRight (76) - : : : :- Project (72) - : : : : +- BroadcastHashJoin Inner BuildLeft (71) - : : : : :- BroadcastExchange (68) - : : : : : +- Project (67) - : : : : : +- Filter (66) - : : : : : +- Scan parquet (65) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (75) - : : : +- Filter (74) - : : : +- Scan parquet (73) - : : +- BroadcastExchange (80) - : : +- Filter (79) - : : +- Scan parquet (78) - : +- BroadcastExchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- BroadcastExchange (90) - +- Filter (89) - +- Scan parquet (88) + Sort (98) + +- Exchange (97) + +- HashAggregate (96) + +- Exchange (95) + +- HashAggregate (94) + +- Project (93) + +- BroadcastHashJoin Inner BuildRight (92) + :- Project (88) + : +- BroadcastHashJoin Inner BuildRight (87) + : :- Project (83) + : : +- BroadcastHashJoin Inner BuildRight (82) + : : :- Project (78) + : : : +- BroadcastHashJoin Inner BuildRight (77) + : : : :- Project (73) + : : : : +- BroadcastHashJoin Inner BuildLeft (72) + : : : : :- BroadcastExchange (69) + : : : : : +- Project (68) + : : : : : +- Filter (67) + : : : : : +- Scan parquet (66) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (76) + : : : +- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- BroadcastExchange (86) + : +- Filter (85) + : +- Scan parquet (84) + +- BroadcastExchange (91) + +- Filter (90) + +- Scan parquet (89) (1) Scan parquet @@ -336,197 +337,201 @@ Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) ColumnarExchange +(58) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(59) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(60) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(60) InputAdapter +(61) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(61) InputIteratorTransformer +(62) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) SortExecTransformer +(63) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(63) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(64) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(67) Project +(68) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(69) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(72) Project +(73) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(77) Project +(78) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(79) Filter +(80) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(80) BroadcastExchange +(81) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(82) Project +(83) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(87) Project +(88) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(88) Scan parquet +(89) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(90) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(90) BroadcastExchange +(91) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(92) Project +(93) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(93) HashAggregate +(94) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(94) Exchange +(95) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) HashAggregate +(96) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(96) Exchange +(97) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(98) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(98) AdaptiveSparkPlan +(99) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt index 9316f6f8ebb5..69a930ea8545 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt index a09e6167ffd3..5d0c6e67cb0d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt @@ -1,78 +1,84 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- TakeOrderedAndProjectExecTransformer (59) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ShuffleQueryStage (36) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ShuffleQueryStage (21) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ShuffleQueryStage (29) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ NoopFilter (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ShuffleQueryStage (44) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (66) + +- TakeOrderedAndProjectExecTransformer (65) + +- ^ ProjectExecTransformer (63) + +- ^ RegularHashAggregateExecTransformer (62) + +- ^ InputIteratorTransformer (61) + +- ShuffleQueryStage (59) + +- ColumnarExchange (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ NoopFilter (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ NoopFilter (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ NoopFilter (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- ShuffledHashJoin Inner BuildRight (81) - :- Exchange (77) - : +- Project (76) - : +- ShuffledHashJoin Inner BuildRight (75) - : :- Exchange (70) - : : +- Project (69) - : : +- ShuffledHashJoin Inner BuildRight (68) - : : :- Exchange (63) - : : : +- Filter (62) - : : : +- Scan parquet (61) - : : +- Exchange (67) - : : +- Project (66) - : : +- Filter (65) - : : +- Scan parquet (64) - : +- Exchange (74) - : +- Project (73) - : +- Filter (72) - : +- Scan parquet (71) - +- Exchange (80) - +- Filter (79) - +- Scan parquet (78) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- ShuffledHashJoin Inner BuildRight (87) + :- Exchange (83) + : +- Project (82) + : +- ShuffledHashJoin Inner BuildRight (81) + : :- Exchange (76) + : : +- Project (75) + : : +- ShuffledHashJoin Inner BuildRight (74) + : : :- Exchange (69) + : : : +- Filter (68) + : : : +- Scan parquet (67) + : : +- Exchange (73) + : : +- Project (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (80) + : +- Project (79) + : +- Filter (78) + : +- Scan parquet (77) + +- Exchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -94,358 +100,382 @@ Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acct Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: X + +(6) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(55) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(56) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(58) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(59) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(60) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(61) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(62) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(57) ProjectExecTransformer +(63) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(58) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(59) TakeOrderedAndProjectExecTransformer +(65) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(60) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(61) Scan parquet +(67) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) Filter +(68) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(63) Exchange +(69) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(65) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(66) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(67) Exchange +(73) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) ShuffledHashJoin +(74) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(69) Project +(75) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(70) Exchange +(76) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(77) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(72) Filter +(78) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(73) Project +(79) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(74) Exchange +(80) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(81) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(76) Project +(82) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(77) Exchange +(83) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(80) Exchange +(86) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(87) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(88) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(89) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(84) Exchange +(90) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(91) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(86) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt index 47b241e9a343..eecba8c53af6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt @@ -1,65 +1,70 @@ == Physical Plan == -AdaptiveSparkPlan (72) +AdaptiveSparkPlan (77) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ SortExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ShuffleQueryStage (38) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (55) + +- ^ SortExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ FilterExecTransformer (46) + +- ^ RegularHashAggregateExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42) + +- ColumnarExchange (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ NoopFilter (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (71) - +- Exchange (70) - +- Filter (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- ShuffledHashJoin Inner BuildRight (64) - :- Exchange (59) - : +- Project (58) - : +- ShuffledHashJoin Inner BuildRight (57) - : :- Exchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- Exchange (56) - : +- Filter (55) - : +- Scan parquet (54) - +- Exchange (63) - +- Project (62) - +- Filter (61) - +- Scan parquet (60) + Sort (76) + +- Exchange (75) + +- Filter (74) + +- HashAggregate (73) + +- Exchange (72) + +- HashAggregate (71) + +- Project (70) + +- ShuffledHashJoin Inner BuildRight (69) + :- Exchange (64) + : +- Project (63) + : +- ShuffledHashJoin Inner BuildRight (62) + : :- Exchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- Exchange (61) + : +- Filter (60) + : +- Scan parquet (59) + +- Exchange (68) + +- Project (67) + +- Filter (66) + +- Scan parquet (65) (1) Scan parquet @@ -81,292 +86,312 @@ Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [1]: [n_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [n_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(34) FlushableHashAggregateExecTransformer +(38) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(41) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(43) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(45) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(42) FilterExecTransformer +(46) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(43) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(44) ColumnarExchange +(48) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(49) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(46) InputAdapter +(51) InputAdapter Input [2]: [ps_partkey#X, value#X] -(47) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(48) SortExecTransformer +(53) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(49) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(50) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(51) Scan parquet +(56) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(52) Filter +(57) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(53) Exchange +(58) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(59) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(60) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) Exchange +(61) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(62) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(58) Project +(63) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(59) Exchange +(64) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(65) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(61) Filter +(66) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(62) Project +(67) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(63) Exchange +(68) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(69) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(65) Project +(70) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(66) HashAggregate +(71) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(67) Exchange +(72) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(73) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(69) Filter +(74) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(70) Exchange +(75) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(76) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(72) AdaptiveSparkPlan +(77) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt index 9adf1b6060ef..e6f809047498 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt @@ -1,46 +1,49 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin Inner BuildLeft (42) - :- Exchange (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- ShuffledHashJoin Inner BuildLeft (45) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -62,196 +65,208 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(31) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) Scan parquet +(38) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(36) Filter +(39) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(37) Exchange +(40) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(41) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(40) Project +(43) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(41) Exchange +(44) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(45) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(43) Project +(46) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(44) HashAggregate +(47) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(45) Exchange +(48) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(49) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(47) Exchange +(50) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(51) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt index fb354643ede9..b716b8f475ba 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt @@ -1,49 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ShuffleQueryStage (5) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftOuter BuildRight (43) - :- Exchange (38) - : +- Scan parquet (37) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftOuter BuildRight (46) + :- Exchange (41) + : +- Scan parquet (40) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -60,223 +63,235 @@ Input [1]: [c_custkey#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(4) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) NoopFilter +(10) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(44) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) Exchange +(45) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(47) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(48) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt index 62a2d4a7b617..00b5fb4142f3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (23) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (25) + +- ^ ProjectExecTransformer (23) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (34) - +- HashAggregate (33) - +- Project (32) - +- ShuffledHashJoin Inner BuildRight (31) - :- Exchange (27) - : +- Project (26) - : +- Filter (25) - : +- Scan parquet (24) - +- Exchange (30) - +- Filter (29) - +- Scan parquet (28) + HashAggregate (36) + +- HashAggregate (35) + +- Project (34) + +- ShuffledHashJoin Inner BuildRight (33) + :- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -52,144 +54,152 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [2]: [p_partkey#X, p_type#X] Arguments: [p_partkey#X, p_type#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(21) ProjectExecTransformer +(23) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(22) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(23) VeloxColumnarToRowExec +(25) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(24) Scan parquet +(26) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(26) Project +(28) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) Exchange +(29) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Scan parquet +(30) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(30) Exchange +(32) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) ShuffledHashJoin +(33) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(32) Project +(34) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(34) HashAggregate +(36) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] -(35) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt index 791ededdabda..dbb20979b70e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt @@ -1,44 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (47) +- == Final Plan == - VeloxColumnarToRowExec (30) - +- ^ SortExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (32) + +- ^ SortExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (22) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- Project (42) - +- ShuffledHashJoin Inner BuildLeft (41) - :- Exchange (33) - : +- Filter (32) - : +- Scan parquet (31) - +- Filter (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- Filter (35) - +- Scan parquet (34) + Sort (46) + +- Exchange (45) + +- Project (44) + +- ShuffledHashJoin Inner BuildLeft (43) + :- Exchange (35) + : +- Filter (34) + : +- Scan parquet (33) + +- Filter (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- Filter (37) + +- Scan parquet (36) (1) Scan parquet @@ -60,182 +62,190 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(16) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(20) FilterExecTransformer +(21) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(21) ShuffledHashJoinExecTransformer +(22) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(26) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) SortExecTransformer +(30) SortExecTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(30) VeloxColumnarToRowExec +(32) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(31) Scan parquet +(33) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(32) Filter +(34) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(33) Exchange +(35) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) Scan parquet +(36) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(35) Filter +(37) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(36) Project +(38) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(37) HashAggregate +(39) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(38) Exchange +(40) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(41) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(40) Filter +(42) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(41) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(42) Project +(44) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(43) Exchange +(45) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(46) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(47) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt index 2060266a2550..e3570601883a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt @@ -1,59 +1,62 @@ == Physical Plan == -AdaptiveSparkPlan (64) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (42) - +- ^ SortExecTransformer (40) - +- ^ InputIteratorTransformer (39) - +- ShuffleQueryStage (37) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (45) + +- ^ SortExecTransformer (43) + +- ^ InputIteratorTransformer (42) + +- ShuffleQueryStage (40) + +- ColumnarExchange (39) + +- VeloxAppendBatches (38) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- ^ ProjectExecTransformer (30) + +- ^ FlushableHashAggregateExecTransformer (29) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (63) - +- Exchange (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- HashAggregate (58) - +- Exchange (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (50) - : +- BroadcastHashJoin LeftAnti BuildRight (49) - : :- Filter (44) - : : +- Scan parquet (43) - : +- BroadcastExchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (53) - +- Filter (52) - +- Scan parquet (51) + Sort (66) + +- Exchange (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- HashAggregate (61) + +- Exchange (60) + +- HashAggregate (59) + +- Project (58) + +- ShuffledHashJoin Inner BuildRight (57) + :- Exchange (53) + : +- BroadcastHashJoin LeftAnti BuildRight (52) + : :- Filter (47) + : : +- Scan parquet (46) + : +- BroadcastExchange (51) + : +- Project (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -75,270 +78,282 @@ Input [2]: [ps_partkey#X, ps_suppkey#X] Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) FlushableHashAggregateExecTransformer +(29) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(30) ColumnarExchange +(32) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(32) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) ColumnarExchange +(38) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(39) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(40) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(38) InputAdapter +(41) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) InputIteratorTransformer +(42) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(40) SortExecTransformer +(43) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(41) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(42) VeloxColumnarToRowExec +(45) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) Scan parquet +(46) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(45) Scan parquet +(48) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(46) Filter +(49) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(47) Project +(50) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(48) BroadcastExchange +(51) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(49) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(50) Exchange +(53) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(54) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(53) Exchange +(56) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(57) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(55) Project +(58) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(56) HashAggregate +(59) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) HashAggregate +(61) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(59) HashAggregate +(62) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(60) Exchange +(63) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(62) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) Sort +(66) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(64) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt index 504ee433e9e8..b79bd948bec8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt @@ -1,56 +1,58 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ ProjectExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) - :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ FilterExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ NoopFilter (20) - +- ^ Scan parquet (19) + VeloxColumnarToRowExec (39) + +- ^ ProjectExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ ProjectExecTransformer (34) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (33) + :- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ NoopFilter (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (56) - +- HashAggregate (55) - +- Project (54) - +- ShuffledHashJoin Inner BuildRight (53) - :- Project (46) - : +- ShuffledHashJoin Inner BuildRight (45) - : :- Exchange (40) - : : +- Filter (39) - : : +- Scan parquet (38) - : +- Exchange (44) - : +- Project (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Filter (48) - +- Scan parquet (47) + HashAggregate (58) + +- HashAggregate (57) + +- Project (56) + +- ShuffledHashJoin Inner BuildRight (55) + :- Project (48) + : +- ShuffledHashJoin Inner BuildRight (47) + : :- Exchange (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Exchange (46) + : +- Project (45) + : +- Filter (44) + : +- Scan parquet (43) + +- Filter (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Filter (50) + +- Scan parquet (49) (1) Scan parquet @@ -72,250 +74,258 @@ Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_container#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(19) Scan parquet +(21) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(20) NoopFilter +(22) NoopFilter Input [2]: [l_partkey#X, l_quantity#X] Arguments: [l_partkey#X, l_quantity#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(30) FilterExecTransformer +(32) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(31) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(32) ProjectExecTransformer +(34) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(33) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(35) ProjectExecTransformer +(37) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(36) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(37) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(38) Scan parquet +(40) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(39) Filter +(41) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(40) Exchange +(42) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(43) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(43) Project +(45) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(44) Exchange +(46) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(46) Project +(48) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(47) Scan parquet +(49) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(48) Filter +(50) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(49) HashAggregate +(51) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(50) Exchange +(52) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) Filter +(54) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(53) ShuffledHashJoin +(55) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(54) Project +(56) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) HashAggregate +(57) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(56) HashAggregate +(58) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] -(57) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt index 78efb8c67470..a7a179a81554 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt @@ -1,89 +1,94 @@ == Physical Plan == -AdaptiveSparkPlan (97) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- TakeOrderedAndProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) - :- ^ InputIteratorTransformer (41) - : +- ShuffleQueryStage (39) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ShuffleQueryStage (32) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ShuffleQueryStage (22) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) - :- ^ InputIteratorTransformer (49) - : +- ShuffleQueryStage (47) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ NoopFilter (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ShuffleQueryStage (51) - +- ReusedExchange (50) + VeloxColumnarToRowExec (69) + +- TakeOrderedAndProjectExecTransformer (68) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ RegularHashAggregateExecTransformer (65) + +- ^ ProjectExecTransformer (64) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (63) + :- ^ InputIteratorTransformer (45) + : +- ShuffleQueryStage (43) + : +- ColumnarExchange (42) + : +- VeloxAppendBatches (41) + : +- ^ ProjectExecTransformer (39) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (38) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (37) + : +- ShuffleQueryStage (35) + : +- ColumnarExchange (34) + : +- VeloxAppendBatches (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (30) + : :- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ NoopFilter (11) + : : +- ^ Scan parquet (10) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ RegularHashAggregateExecTransformer (27) + : +- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24) + : +- ColumnarExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) + :- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52) + : +- ColumnarExchange (51) + : +- VeloxAppendBatches (50) + : +- ^ ProjectExecTransformer (48) + : +- ^ NoopFilter (47) + : +- ^ Scan parquet (46) + +- ^ ProjectExecTransformer (61) + +- ^ FilterExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56) + +- ReusedExchange (55) +- == Initial Plan == - TakeOrderedAndProject (96) - +- HashAggregate (95) - +- HashAggregate (94) - +- Project (93) - +- ShuffledHashJoin Inner BuildRight (92) - :- Exchange (81) - : +- Project (80) - : +- ShuffledHashJoin Inner BuildLeft (79) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (78) - : +- ShuffledHashJoin LeftSemi BuildRight (77) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Project (76) - : +- Filter (75) - : +- HashAggregate (74) - : +- Exchange (73) - : +- HashAggregate (72) - : +- Scan parquet (71) - +- ShuffledHashJoin LeftSemi BuildRight (91) - :- Exchange (84) - : +- Filter (83) - : +- Scan parquet (82) - +- Project (90) - +- Filter (89) - +- HashAggregate (88) - +- Exchange (87) - +- HashAggregate (86) - +- Scan parquet (85) + TakeOrderedAndProject (101) + +- HashAggregate (100) + +- HashAggregate (99) + +- Project (98) + +- ShuffledHashJoin Inner BuildRight (97) + :- Exchange (86) + : +- Project (85) + : +- ShuffledHashJoin Inner BuildLeft (84) + : :- Exchange (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (83) + : +- ShuffledHashJoin LeftSemi BuildRight (82) + : :- Exchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- Project (81) + : +- Filter (80) + : +- HashAggregate (79) + : +- Exchange (78) + : +- HashAggregate (77) + : +- Scan parquet (76) + +- ShuffledHashJoin LeftSemi BuildRight (96) + :- Exchange (89) + : +- Filter (88) + : +- Scan parquet (87) + +- Project (95) + +- Filter (94) + +- HashAggregate (93) + +- Exchange (92) + +- HashAggregate (91) + +- Scan parquet (90) (1) Scan parquet @@ -105,420 +110,440 @@ Input [2]: [c_custkey#X, c_name#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(26) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(27) ProjectExecTransformer +(29) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(28) ShuffledHashJoinExecTransformer +(30) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(30) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(31) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(33) InputAdapter +(36) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(34) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) ShuffledHashJoinExecTransformer +(38) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(36) ProjectExecTransformer +(39) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(38) ColumnarExchange +(41) VeloxAppendBatches +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(42) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(43) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(40) InputAdapter +(44) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(41) InputIteratorTransformer +(45) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(42) Scan parquet +(46) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(43) NoopFilter +(47) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(46) ColumnarExchange +(50) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: X + +(51) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(52) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(48) InputAdapter +(53) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(49) InputIteratorTransformer +(54) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(50) ReusedExchange [Reuses operator id: 21] +(55) ReusedExchange [Reuses operator id: 23] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) ShuffleQueryStage +(56) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(52) InputAdapter +(57) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(53) InputIteratorTransformer +(58) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(54) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(55) FilterExecTransformer +(60) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(56) ProjectExecTransformer +(61) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(58) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(59) ProjectExecTransformer +(64) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(60) RegularHashAggregateExecTransformer +(65) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(61) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(62) WholeStageCodegenTransformer (X) +(67) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(63) TakeOrderedAndProjectExecTransformer +(68) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(64) VeloxColumnarToRowExec +(69) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(65) Scan parquet +(70) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(66) Filter +(71) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(67) Exchange +(72) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(73) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(69) Filter +(74) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(70) Exchange +(75) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(76) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(77) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(78) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(79) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(80) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(81) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Exchange +(83) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) ShuffledHashJoin +(84) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(80) Project +(85) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(81) Exchange +(86) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(87) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(83) Filter +(88) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(84) Exchange +(89) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(90) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(86) HashAggregate +(91) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(87) Exchange +(92) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) HashAggregate +(93) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(89) Filter +(94) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(90) Project +(95) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(91) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(93) Project +(98) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(94) HashAggregate +(99) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(95) HashAggregate +(100) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(96) TakeOrderedAndProject +(101) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(97) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt index aa13d1509c29..6ec9ae965ee9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (33) - +- HashAggregate (32) - +- Project (31) - +- ShuffledHashJoin Inner BuildRight (30) - :- Exchange (26) - : +- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- Exchange (29) - +- Filter (28) - +- Scan parquet (27) + HashAggregate (35) + +- HashAggregate (34) + +- Project (33) + +- ShuffledHashJoin Inner BuildRight (32) + :- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) (1) Scan parquet @@ -51,140 +53,148 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(24) VeloxColumnarToRowExec Input [1]: [revenue#X] -(23) Scan parquet +(25) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(24) Filter +(26) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(25) Project +(27) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(26) Exchange +(28) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Scan parquet +(29) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(28) Filter +(30) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(29) Exchange +(31) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) ShuffledHashJoin +(32) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(31) Project +(33) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(32) HashAggregate +(34) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(34) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt index 959bcd4ef703..24be4842e1b8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt @@ -1,110 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (126) +AdaptiveSparkPlan (136) +- == Final Plan == - VeloxColumnarToRowExec (86) - +- ^ SortExecTransformer (84) - +- ^ InputIteratorTransformer (83) - +- ShuffleQueryStage (81) - +- ColumnarExchange (80) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ShuffleQueryStage (29) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ShuffleQueryStage (14) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ NoopFilter (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ShuffleQueryStage (22) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ NoopFilter (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ NoopFilter (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41) - : +- ReusedExchange (40) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (96) + +- ^ SortExecTransformer (94) + +- ^ InputIteratorTransformer (93) + +- ShuffleQueryStage (91) + +- ColumnarExchange (90) + +- VeloxAppendBatches (89) + +- ^ ProjectExecTransformer (87) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (86) + :- ^ InputIteratorTransformer (76) + : +- ShuffleQueryStage (74) + : +- ColumnarExchange (73) + : +- VeloxAppendBatches (72) + : +- ^ ProjectExecTransformer (70) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (69) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- VeloxAppendBatches (64) + : +- ^ ProjectExecTransformer (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (61) + : :- ^ InputIteratorTransformer (35) + : : +- ShuffleQueryStage (33) + : : +- ColumnarExchange (32) + : : +- VeloxAppendBatches (31) + : : +- ^ ProjectExecTransformer (29) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : : :- ^ InputIteratorTransformer (18) + : : : +- ShuffleQueryStage (16) + : : : +- ColumnarExchange (15) + : : : +- VeloxAppendBatches (14) + : : : +- ^ ProjectExecTransformer (12) + : : : +- ^ NoopFilter (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (27) + : : +- ShuffleQueryStage (25) + : : +- ColumnarExchange (24) + : : +- VeloxAppendBatches (23) + : : +- ^ ProjectExecTransformer (21) + : : +- ^ NoopFilter (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (60) + : +- ShuffleQueryStage (58) + : +- ColumnarExchange (57) + : +- VeloxAppendBatches (56) + : +- ^ ProjectExecTransformer (54) + : +- ^ FilterExecTransformer (53) + : +- ^ ProjectExecTransformer (52) + : +- ^ RegularHashAggregateExecTransformer (51) + : +- ^ RegularHashAggregateExecTransformer (50) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (49) + : :- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ NoopFilter (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (48) + : +- ShuffleQueryStage (46) + : +- ReusedExchange (45) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83) + +- ColumnarExchange (82) + +- VeloxAppendBatches (81) + +- ^ ProjectExecTransformer (79) + +- ^ NoopFilter (78) + +- ^ Scan parquet (77) +- == Initial Plan == - Sort (125) - +- Exchange (124) - +- Project (123) - +- ShuffledHashJoin Inner BuildRight (122) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin LeftSemi BuildRight (115) - : :- Exchange (89) - : : +- Filter (88) - : : +- Scan parquet (87) - : +- Exchange (114) - : +- Project (113) - : +- ShuffledHashJoin Inner BuildLeft (112) - : :- Exchange (98) - : : +- ShuffledHashJoin LeftSemi BuildRight (97) - : : :- Exchange (92) - : : : +- Filter (91) - : : : +- Scan parquet (90) - : : +- Exchange (96) - : : +- Project (95) - : : +- Filter (94) - : : +- Scan parquet (93) - : +- Exchange (111) - : +- Filter (110) - : +- HashAggregate (109) - : +- HashAggregate (108) - : +- ShuffledHashJoin LeftSemi BuildRight (107) - : :- Exchange (102) - : : +- Project (101) - : : +- Filter (100) - : : +- Scan parquet (99) - : +- Exchange (106) - : +- Project (105) - : +- Filter (104) - : +- Scan parquet (103) - +- Exchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (135) + +- Exchange (134) + +- Project (133) + +- ShuffledHashJoin Inner BuildRight (132) + :- Exchange (127) + : +- Project (126) + : +- ShuffledHashJoin LeftSemi BuildRight (125) + : :- Exchange (99) + : : +- Filter (98) + : : +- Scan parquet (97) + : +- Exchange (124) + : +- Project (123) + : +- ShuffledHashJoin Inner BuildLeft (122) + : :- Exchange (108) + : : +- ShuffledHashJoin LeftSemi BuildRight (107) + : : :- Exchange (102) + : : : +- Filter (101) + : : : +- Scan parquet (100) + : : +- Exchange (106) + : : +- Project (105) + : : +- Filter (104) + : : +- Scan parquet (103) + : +- Exchange (121) + : +- Filter (120) + : +- HashAggregate (119) + : +- HashAggregate (118) + : +- ShuffledHashJoin LeftSemi BuildRight (117) + : :- Exchange (112) + : : +- Project (111) + : : +- Filter (110) + : : +- Scan parquet (109) + : +- Exchange (116) + : +- Project (115) + : +- Filter (114) + : +- Scan parquet (113) + +- Exchange (131) + +- Project (130) + +- Filter (129) + +- Scan parquet (128) (1) Scan parquet @@ -126,518 +136,558 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(18) NoopFilter +(20) NoopFilter Input [2]: [p_partkey#X, p_name#X] Arguments: [p_partkey#X, p_name#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [1]: [p_partkey#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [1]: [p_partkey#X] -(25) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(32) Scan parquet +(36) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(33) NoopFilter +(37) NoopFilter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(40) ReusedExchange [Reuses operator id: 21] +(45) ReusedExchange [Reuses operator id: 24] Output [1]: [p_partkey#X] -(41) ShuffleQueryStage +(46) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(42) InputAdapter +(47) InputAdapter Input [1]: [p_partkey#X] -(43) InputIteratorTransformer +(48) InputIteratorTransformer Input [1]: [p_partkey#X] -(44) ShuffledHashJoinExecTransformer +(49) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(45) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(46) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(47) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(48) FilterExecTransformer +(53) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(49) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(51) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(53) InputAdapter +(59) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(54) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(55) ShuffledHashJoinExecTransformer +(61) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(58) ColumnarExchange +(64) VeloxAppendBatches +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: X + +(65) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(66) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(60) InputAdapter +(67) InputAdapter Input [1]: [ps_suppkey#X] -(61) InputIteratorTransformer +(68) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(62) ShuffledHashJoinExecTransformer +(69) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(63) ProjectExecTransformer +(70) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(71) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(72) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(73) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(74) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(75) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(68) InputIteratorTransformer +(76) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(69) Scan parquet +(77) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(70) NoopFilter +(78) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(71) ProjectExecTransformer +(79) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(73) ColumnarExchange +(81) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(82) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(83) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(75) InputAdapter +(84) InputAdapter Input [1]: [n_nationkey#X] -(76) InputIteratorTransformer +(85) InputIteratorTransformer Input [1]: [n_nationkey#X] -(77) ShuffledHashJoinExecTransformer +(86) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(87) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(79) WholeStageCodegenTransformer (X) +(88) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(80) ColumnarExchange +(89) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(90) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(91) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(82) InputAdapter +(92) InputAdapter Input [2]: [s_name#X, s_address#X] -(83) InputIteratorTransformer +(93) InputIteratorTransformer Input [2]: [s_name#X, s_address#X] -(84) SortExecTransformer +(94) SortExecTransformer Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(85) WholeStageCodegenTransformer (X) +(95) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(86) VeloxColumnarToRowExec +(96) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(87) Scan parquet +(97) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(88) Filter +(98) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(89) Exchange +(99) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(100) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(91) Filter +(101) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(92) Exchange +(102) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Scan parquet +(103) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(94) Filter +(104) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(95) Project +(105) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(96) Exchange +(106) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(98) Exchange +(108) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(109) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(100) Filter +(110) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(101) Project +(111) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(102) Exchange +(112) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) Scan parquet +(113) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(104) Filter +(114) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(105) Project +(115) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(106) Exchange +(116) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(117) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(108) HashAggregate +(118) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(109) HashAggregate +(119) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(110) Filter +(120) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(111) Exchange +(121) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(113) Project +(123) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(114) Exchange +(124) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(116) Project +(126) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(117) Exchange +(127) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(128) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(119) Filter +(129) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(120) Project +(130) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(121) Exchange +(131) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(123) Project +(133) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(124) Exchange +(134) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) Sort +(135) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(126) AdaptiveSparkPlan +(136) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt index e9418fc2a71c..0099ce88edc8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt @@ -1,104 +1,113 @@ == Physical Plan == -AdaptiveSparkPlan (119) +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (82) - +- TakeOrderedAndProjectExecTransformer (81) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ShuffleQueryStage (21) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ShuffleQueryStage (30) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ NoopFilter (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ NoopFilter (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ShuffleQueryStage (67) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ NoopFilter (63) - +- ^ Scan parquet (62) + VeloxColumnarToRowExec (91) + +- TakeOrderedAndProjectExecTransformer (90) + +- ^ RegularHashAggregateExecTransformer (88) + +- ^ InputIteratorTransformer (87) + +- ShuffleQueryStage (85) + +- ColumnarExchange (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ NoopFilter (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ NoopFilter (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ NoopFilter (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (118) - +- HashAggregate (117) - +- Exchange (116) - +- HashAggregate (115) - +- Project (114) - +- ShuffledHashJoin Inner BuildRight (113) - :- Exchange (108) - : +- Project (107) - : +- ShuffledHashJoin Inner BuildRight (106) - : :- Exchange (101) - : : +- Project (100) - : : +- ShuffledHashJoin Inner BuildLeft (99) - : : :- Exchange (85) - : : : +- Filter (84) - : : : +- Scan parquet (83) - : : +- Exchange (98) - : : +- ShuffledHashJoin LeftAnti BuildRight (97) - : : :- ShuffledHashJoin LeftSemi BuildRight (92) - : : : :- Exchange (89) - : : : : +- Project (88) - : : : : +- Filter (87) - : : : : +- Scan parquet (86) - : : : +- Exchange (91) - : : : +- Scan parquet (90) - : : +- Exchange (96) - : : +- Project (95) - : : +- Filter (94) - : : +- Scan parquet (93) - : +- Exchange (105) - : +- Project (104) - : +- Filter (103) - : +- Scan parquet (102) - +- Exchange (112) - +- Project (111) - +- Filter (110) - +- Scan parquet (109) + TakeOrderedAndProject (127) + +- HashAggregate (126) + +- Exchange (125) + +- HashAggregate (124) + +- Project (123) + +- ShuffledHashJoin Inner BuildRight (122) + :- Exchange (117) + : +- Project (116) + : +- ShuffledHashJoin Inner BuildRight (115) + : :- Exchange (110) + : : +- Project (109) + : : +- ShuffledHashJoin Inner BuildLeft (108) + : : :- Exchange (94) + : : : +- Filter (93) + : : : +- Scan parquet (92) + : : +- Exchange (107) + : : +- ShuffledHashJoin LeftAnti BuildRight (106) + : : :- ShuffledHashJoin LeftSemi BuildRight (101) + : : : :- Exchange (98) + : : : : +- Project (97) + : : : : +- Filter (96) + : : : : +- Scan parquet (95) + : : : +- Exchange (100) + : : : +- Scan parquet (99) + : : +- Exchange (105) + : : +- Project (104) + : : +- Filter (103) + : : +- Scan parquet (102) + : +- Exchange (114) + : +- Project (113) + : +- Filter (112) + : +- Scan parquet (111) + +- Exchange (121) + +- Project (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -120,494 +129,530 @@ Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(24) ShuffledHashJoinExecTransformer +(27) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(25) Scan parquet +(28) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(26) NoopFilter +(29) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(29) ColumnarExchange +(32) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(33) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(33) ShuffledHashJoinExecTransformer +(37) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(41) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(40) ShuffledHashJoinExecTransformer +(45) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(47) Scan parquet +(53) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(48) NoopFilter +(54) NoopFilter Input [2]: [o_orderkey#X, o_orderstatus#X] Arguments: [o_orderkey#X, o_orderstatus#X] -(49) ProjectExecTransformer +(55) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(50) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(51) ColumnarExchange +(57) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: X + +(58) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(53) InputAdapter +(60) InputAdapter Input [1]: [o_orderkey#X] -(54) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [o_orderkey#X] -(55) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(62) Scan parquet +(70) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(63) NoopFilter +(71) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(64) ProjectExecTransformer +(72) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(65) WholeStageCodegenTransformer (X) +(73) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(66) ColumnarExchange +(74) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(75) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(76) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(68) InputAdapter +(77) InputAdapter Input [1]: [n_nationkey#X] -(69) InputIteratorTransformer +(78) InputIteratorTransformer Input [1]: [n_nationkey#X] -(70) ShuffledHashJoinExecTransformer +(79) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(72) FlushableHashAggregateExecTransformer +(81) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(73) ProjectExecTransformer +(82) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(74) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(75) ColumnarExchange +(84) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(85) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(77) InputAdapter +(86) InputAdapter Input [2]: [s_name#X, count#X] -(78) InputIteratorTransformer +(87) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(79) RegularHashAggregateExecTransformer +(88) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(80) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(81) TakeOrderedAndProjectExecTransformer +(90) TakeOrderedAndProjectExecTransformer Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X], 0 -(82) VeloxColumnarToRowExec +(91) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(83) Scan parquet +(92) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(84) Filter +(93) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(85) Exchange +(94) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(86) Scan parquet +(95) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(87) Filter +(96) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(88) Project +(97) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(89) Exchange +(98) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(99) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(91) Exchange +(100) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) ShuffledHashJoin +(101) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(93) Scan parquet +(102) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(94) Filter +(103) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(95) Project +(104) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(96) Exchange +(105) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(106) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(98) Exchange +(107) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(100) Project +(109) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(101) Exchange +(110) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(103) Filter +(112) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(104) Project +(113) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(105) Exchange +(114) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) ShuffledHashJoin +(115) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(107) Project +(116) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(108) Exchange +(117) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Scan parquet +(118) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(110) Filter +(119) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(111) Project +(120) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(112) Exchange +(121) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(114) Project +(123) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(115) HashAggregate +(124) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(116) Exchange +(125) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) HashAggregate +(126) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(118) TakeOrderedAndProject +(127) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(119) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt index 2b0fcd16aadc..ac5dfa762fa1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt @@ -1,43 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (46) +AdaptiveSparkPlan (49) +- == Final Plan == - VeloxColumnarToRowExec (33) - +- ^ SortExecTransformer (31) - +- ^ InputIteratorTransformer (30) - +- ShuffleQueryStage (28) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (45) - +- Exchange (44) - +- HashAggregate (43) - +- Exchange (42) - +- HashAggregate (41) - +- Project (40) - +- ShuffledHashJoin LeftAnti BuildRight (39) - :- Exchange (36) - : +- Filter (35) - : +- Scan parquet (34) - +- Exchange (38) - +- Scan parquet (37) + Sort (48) + +- Exchange (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- ShuffledHashJoin LeftAnti BuildRight (42) + :- Exchange (39) + : +- Filter (38) + : +- Scan parquet (37) + +- Exchange (41) + +- Scan parquet (40) (1) Scan parquet @@ -59,182 +62,194 @@ Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) Scan parquet +(10) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [1]: [o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [1]: [o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(26) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) ColumnarExchange +(29) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(30) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(28) ShuffleQueryStage +(31) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(29) InputAdapter +(32) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) InputIteratorTransformer +(33) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(31) SortExecTransformer +(34) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(32) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(33) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) Scan parquet +(37) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(35) Filter +(38) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(36) Exchange +(39) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(40) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(40) Project +(43) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(41) HashAggregate +(44) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(42) Exchange +(45) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) HashAggregate +(46) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(44) Exchange +(47) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(48) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(46) AdaptiveSparkPlan +(49) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt index ed7b3ea7d377..f188fa96b0d8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt @@ -1,56 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ NoopFilter (28) + +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (49) - : +- Project (48) - : +- ShuffledHashJoin Inner BuildLeft (47) - : :- Exchange (43) - : : +- Project (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Exchange (53) - +- Project (52) - +- Filter (51) - +- Scan parquet (50) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Project (59) + +- ShuffledHashJoin Inner BuildRight (58) + :- Exchange (53) + : +- Project (52) + : +- ShuffledHashJoin Inner BuildLeft (51) + : :- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Exchange (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -72,244 +76,260 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(23) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(42) Project +(46) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(43) Exchange +(47) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(48) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(45) Filter +(49) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(46) Exchange +(50) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(51) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(48) Project +(52) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Exchange +(53) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(54) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(51) Filter +(55) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(52) Project +(56) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(53) Exchange +(57) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(58) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(55) Project +(59) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(56) HashAggregate +(60) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(57) HashAggregate +(61) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt index 07a00316284a..0a7ae2856c03 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt @@ -1,47 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftSemi BuildRight (43) - :- Exchange (38) - : +- Project (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftSemi BuildRight (46) + :- Exchange (41) + : +- Project (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -63,200 +66,212 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [l_orderkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [l_orderkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(35) Scan parquet +(38) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(36) Filter +(39) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(37) Project +(40) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(38) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(41) Project +(44) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(42) Exchange +(45) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(44) Project +(47) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(45) HashAggregate +(48) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(46) Exchange +(49) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(50) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(48) Exchange +(51) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(52) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt index 766f9f5a1314..8b26063b1527 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt @@ -1,115 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (134) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (133) - +- Exchange (132) - +- HashAggregate (131) - +- Exchange (130) - +- HashAggregate (129) - +- Project (128) - +- ShuffledHashJoin Inner BuildRight (127) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Project (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (126) - +- Project (125) - +- Filter (124) - +- Scan parquet (123) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Project (111) + : : : : +- Filter (110) + : : : : +- Scan parquet (109) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (137) + +- Project (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -131,552 +142,596 @@ Input [2]: [c_custkey#X, c_nationkey#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X + +(66) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: X + +(74) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [1]: [r_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [1]: [r_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(99) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [2]: [n_name#X, revenue#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) Exchange +(108) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(109) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(99) Filter +(110) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(100) Project +(111) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(101) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(103) Project +(114) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(104) Exchange +(115) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(107) Exchange +(118) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(109) Project +(120) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(110) Exchange +(121) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(113) Exchange +(124) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(115) Project +(126) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(116) Exchange +(127) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(119) Exchange +(130) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(121) Project +(132) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(122) Exchange +(133) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(125) Project +(136) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(126) Exchange +(137) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(128) Project +(139) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(129) HashAggregate +(140) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(130) Exchange +(141) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) HashAggregate +(142) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(132) Exchange +(143) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(144) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(134) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt index 7aae3dccfd9b..a5022343944f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt index 67c25166ae14..a7a157760c7c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt @@ -1,110 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (138) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ SortExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ShuffleQueryStage (70) - +- ReusedExchange (69) + VeloxColumnarToRowExec (100) + +- ^ SortExecTransformer (98) + +- ^ InputIteratorTransformer (97) + +- ShuffleQueryStage (95) + +- ColumnarExchange (94) + +- VeloxAppendBatches (93) + +- ^ RegularHashAggregateExecTransformer (91) + +- ^ InputIteratorTransformer (90) + +- ShuffleQueryStage (88) + +- ColumnarExchange (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79) + +- ReusedExchange (78) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (111) - : : +- Project (110) - : : +- ShuffledHashJoin Inner BuildRight (109) - : : :- Exchange (105) - : : : +- Project (104) - : : : +- ShuffledHashJoin Inner BuildRight (103) - : : : :- Exchange (99) - : : : : +- Project (98) - : : : : +- ShuffledHashJoin Inner BuildLeft (97) - : : : : :- Exchange (93) - : : : : : +- Filter (92) - : : : : : +- Scan parquet (91) - : : : : +- Exchange (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Exchange (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (114) - : +- Filter (113) - : +- Scan parquet (112) - +- Exchange (120) - +- Filter (119) - +- Scan parquet (118) + Sort (137) + +- Exchange (136) + +- HashAggregate (135) + +- Exchange (134) + +- HashAggregate (133) + +- Project (132) + +- ShuffledHashJoin Inner BuildRight (131) + :- Exchange (127) + : +- Project (126) + : +- ShuffledHashJoin Inner BuildRight (125) + : :- Exchange (121) + : : +- Project (120) + : : +- ShuffledHashJoin Inner BuildRight (119) + : : :- Exchange (115) + : : : +- Project (114) + : : : +- ShuffledHashJoin Inner BuildRight (113) + : : : :- Exchange (109) + : : : : +- Project (108) + : : : : +- ShuffledHashJoin Inner BuildLeft (107) + : : : : :- Exchange (103) + : : : : : +- Filter (102) + : : : : : +- Scan parquet (101) + : : : : +- Exchange (106) + : : : : +- Filter (105) + : : : : +- Scan parquet (104) + : : : +- Exchange (112) + : : : +- Filter (111) + : : : +- Scan parquet (110) + : : +- Exchange (118) + : : +- Filter (117) + : : +- Scan parquet (116) + : +- Exchange (124) + : +- Filter (123) + : +- Scan parquet (122) + +- Exchange (130) + +- Filter (129) + +- Scan parquet (128) (1) Scan parquet @@ -126,524 +136,564 @@ Input [2]: [s_suppkey#X, s_nationkey#X] Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(23) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_orderkey#X, o_custkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: X + +(40) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(69) ReusedExchange [Reuses operator id: 58] +(78) ReusedExchange [Reuses operator id: 66] Output [2]: [n_nationkey#X, n_name#X] -(70) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(71) InputAdapter +(80) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(72) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(73) ShuffledHashJoinExecTransformer +(82) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(74) ProjectExecTransformer +(83) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(75) FlushableHashAggregateExecTransformer +(84) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) ProjectExecTransformer +(85) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) WholeStageCodegenTransformer (X) +(86) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(78) ColumnarExchange +(87) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(88) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(80) InputAdapter +(89) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(81) InputIteratorTransformer +(90) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(82) RegularHashAggregateExecTransformer +(91) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(83) WholeStageCodegenTransformer (X) +(92) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(84) ColumnarExchange +(93) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(94) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(95) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(86) InputAdapter +(96) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(87) InputIteratorTransformer +(97) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(88) SortExecTransformer +(98) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(89) WholeStageCodegenTransformer (X) +(99) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(90) VeloxColumnarToRowExec +(100) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(91) Scan parquet +(101) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(102) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(103) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(104) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(105) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Exchange +(106) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(98) Project +(108) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(102) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(104) Project +(114) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(105) Exchange +(115) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(116) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(117) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(118) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(110) Project +(120) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(111) Exchange +(121) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(122) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(113) Filter +(123) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(114) Exchange +(124) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(116) Project +(126) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(117) Exchange +(127) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(128) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(119) Filter +(129) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(120) Exchange +(130) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(122) Project +(132) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(123) HashAggregate +(133) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(124) Exchange +(134) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(135) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(126) Exchange +(136) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(137) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(128) AdaptiveSparkPlan +(138) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt index 4c1fb04b8660..240315c7973c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt @@ -1,150 +1,165 @@ == Physical Plan == -AdaptiveSparkPlan (177) +AdaptiveSparkPlan (192) +- == Final Plan == - VeloxColumnarToRowExec (125) - +- ^ SortExecTransformer (123) - +- ^ InputIteratorTransformer (122) - +- ShuffleQueryStage (120) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ShuffleQueryStage (113) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ShuffleQueryStage (96) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ShuffleQueryStage (81) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ShuffleQueryStage (66) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ShuffleQueryStage (51) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ShuffleQueryStage (36) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ShuffleQueryStage (21) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ShuffleQueryStage (6) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ShuffleQueryStage (14) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ShuffleQueryStage (29) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ NoopFilter (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ShuffleQueryStage (44) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ NoopFilter (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ShuffleQueryStage (59) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ NoopFilter (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ShuffleQueryStage (74) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ NoopFilter (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ NoopFilter (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ShuffleQueryStage (104) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ NoopFilter (100) - +- ^ Scan parquet (99) + VeloxColumnarToRowExec (140) + +- ^ SortExecTransformer (138) + +- ^ InputIteratorTransformer (137) + +- ShuffleQueryStage (135) + +- ColumnarExchange (134) + +- VeloxAppendBatches (133) + +- ^ ProjectExecTransformer (131) + +- ^ RegularHashAggregateExecTransformer (130) + +- ^ InputIteratorTransformer (129) + +- ShuffleQueryStage (127) + +- ColumnarExchange (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ NoopFilter (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ NoopFilter (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ NoopFilter (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ NoopFilter (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ NoopFilter (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ NoopFilter (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ NoopFilter (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (176) - +- Exchange (175) - +- HashAggregate (174) - +- Exchange (173) - +- HashAggregate (172) - +- Project (171) - +- ShuffledHashJoin Inner BuildRight (170) - :- Exchange (165) - : +- Project (164) - : +- ShuffledHashJoin Inner BuildRight (163) - : :- Exchange (159) - : : +- Project (158) - : : +- ShuffledHashJoin Inner BuildRight (157) - : : :- Exchange (153) - : : : +- Project (152) - : : : +- ShuffledHashJoin Inner BuildRight (151) - : : : :- Exchange (147) - : : : : +- Project (146) - : : : : +- ShuffledHashJoin Inner BuildRight (145) - : : : : :- Exchange (141) - : : : : : +- Project (140) - : : : : : +- ShuffledHashJoin Inner BuildRight (139) - : : : : : :- Exchange (135) - : : : : : : +- Project (134) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) - : : : : : : :- Exchange (129) - : : : : : : : +- Project (128) - : : : : : : : +- Filter (127) - : : : : : : : +- Scan parquet (126) - : : : : : : +- Exchange (132) - : : : : : : +- Filter (131) - : : : : : : +- Scan parquet (130) - : : : : : +- Exchange (138) - : : : : : +- Filter (137) - : : : : : +- Scan parquet (136) - : : : : +- Exchange (144) - : : : : +- Filter (143) - : : : : +- Scan parquet (142) - : : : +- Exchange (150) - : : : +- Filter (149) - : : : +- Scan parquet (148) - : : +- Exchange (156) - : : +- Filter (155) - : : +- Scan parquet (154) - : +- Exchange (162) - : +- Filter (161) - : +- Scan parquet (160) - +- Exchange (169) - +- Project (168) - +- Filter (167) - +- Scan parquet (166) + Sort (191) + +- Exchange (190) + +- HashAggregate (189) + +- Exchange (188) + +- HashAggregate (187) + +- Project (186) + +- ShuffledHashJoin Inner BuildRight (185) + :- Exchange (180) + : +- Project (179) + : +- ShuffledHashJoin Inner BuildRight (178) + : :- Exchange (174) + : : +- Project (173) + : : +- ShuffledHashJoin Inner BuildRight (172) + : : :- Exchange (168) + : : : +- Project (167) + : : : +- ShuffledHashJoin Inner BuildRight (166) + : : : :- Exchange (162) + : : : : +- Project (161) + : : : : +- ShuffledHashJoin Inner BuildRight (160) + : : : : :- Exchange (156) + : : : : : +- Project (155) + : : : : : +- ShuffledHashJoin Inner BuildRight (154) + : : : : : :- Exchange (150) + : : : : : : +- Project (149) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (148) + : : : : : : :- Exchange (144) + : : : : : : : +- Project (143) + : : : : : : : +- Filter (142) + : : : : : : : +- Scan parquet (141) + : : : : : : +- Exchange (147) + : : : : : : +- Filter (146) + : : : : : : +- Scan parquet (145) + : : : : : +- Exchange (153) + : : : : : +- Filter (152) + : : : : : +- Scan parquet (151) + : : : : +- Exchange (159) + : : : : +- Filter (158) + : : : : +- Scan parquet (157) + : : : +- Exchange (165) + : : : +- Filter (164) + : : : +- Scan parquet (163) + : : +- Exchange (171) + : : +- Filter (170) + : : +- Scan parquet (169) + : +- Exchange (177) + : +- Filter (176) + : +- Scan parquet (175) + +- Exchange (184) + +- Project (183) + +- Filter (182) + +- Scan parquet (181) (1) Scan parquet @@ -166,732 +181,792 @@ Input [2]: [p_partkey#X, p_type#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: [n_nationkey#X, n_regionkey#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(79) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(80) ColumnarExchange +(90) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: X + +(91) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(92) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(82) InputAdapter +(93) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(83) InputIteratorTransformer +(94) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(84) Scan parquet +(95) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) NoopFilter +(96) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(86) ProjectExecTransformer +(97) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(92) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(93) ProjectExecTransformer +(105) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(94) WholeStageCodegenTransformer (X) +(106) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(95) ColumnarExchange +(107) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: X + +(108) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(109) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(97) InputAdapter +(110) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(98) InputIteratorTransformer +(111) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(99) Scan parquet +(112) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(100) NoopFilter +(113) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(101) ProjectExecTransformer +(114) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(102) WholeStageCodegenTransformer (X) +(115) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(103) ColumnarExchange +(116) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(117) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(104) ShuffleQueryStage +(118) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(105) InputAdapter +(119) InputAdapter Input [1]: [r_regionkey#X] -(106) InputIteratorTransformer +(120) InputIteratorTransformer Input [1]: [r_regionkey#X] -(107) ShuffledHashJoinExecTransformer +(121) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(108) ProjectExecTransformer +(122) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(109) FlushableHashAggregateExecTransformer +(123) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(110) ProjectExecTransformer +(124) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(111) WholeStageCodegenTransformer (X) +(125) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(112) ColumnarExchange +(126) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(113) ShuffleQueryStage +(127) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(114) InputAdapter +(128) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(115) InputIteratorTransformer +(129) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(116) RegularHashAggregateExecTransformer +(130) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(117) ProjectExecTransformer +(131) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(118) WholeStageCodegenTransformer (X) +(132) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(119) ColumnarExchange +(133) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(134) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(120) ShuffleQueryStage +(135) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(121) InputAdapter +(136) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(122) InputIteratorTransformer +(137) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(123) SortExecTransformer +(138) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(124) WholeStageCodegenTransformer (X) +(139) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(125) VeloxColumnarToRowExec +(140) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(126) Scan parquet +(141) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(127) Filter +(142) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(128) Project +(143) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(129) Exchange +(144) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) Scan parquet +(145) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(131) Filter +(146) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(132) Exchange +(147) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) ShuffledHashJoin +(148) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(134) Project +(149) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(135) Exchange +(150) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Scan parquet +(151) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(137) Filter +(152) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(138) Exchange +(153) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(154) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(140) Project +(155) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(141) Exchange +(156) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) Scan parquet +(157) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(143) Filter +(158) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(144) Exchange +(159) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) ShuffledHashJoin +(160) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(146) Project +(161) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(147) Exchange +(162) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Scan parquet +(163) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(149) Filter +(164) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(150) Exchange +(165) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) ShuffledHashJoin +(166) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(152) Project +(167) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(153) Exchange +(168) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) Scan parquet +(169) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(155) Filter +(170) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(156) Exchange +(171) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) ShuffledHashJoin +(172) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(158) Project +(173) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(159) Exchange +(174) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Scan parquet +(175) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(161) Filter +(176) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(162) Exchange +(177) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) ShuffledHashJoin +(178) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(164) Project +(179) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(165) Exchange +(180) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) Scan parquet +(181) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(167) Filter +(182) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(168) Project +(183) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(169) Exchange +(184) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) ShuffledHashJoin +(185) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(171) Project +(186) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(172) HashAggregate +(187) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(173) Exchange +(188) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(174) HashAggregate +(189) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(175) Exchange +(190) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(191) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(177) AdaptiveSparkPlan +(192) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt index 582d323dccb2..2bbb889c2ae3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt @@ -1,114 +1,125 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (144) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- HashAggregate (130) - +- Exchange (129) - +- HashAggregate (128) - +- Project (127) - +- ShuffledHashJoin Inner BuildRight (126) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (98) - : : : : : +- Project (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (125) - +- Filter (124) - +- Scan parquet (123) + Sort (143) + +- Exchange (142) + +- HashAggregate (141) + +- Exchange (140) + +- HashAggregate (139) + +- Project (138) + +- ShuffledHashJoin Inner BuildRight (137) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (109) + : : : : : +- Project (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -130,548 +141,592 @@ Input [2]: [p_partkey#X, p_name#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: X + +(57) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_orderdate#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: X + +(74) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(99) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(97) Project +(108) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(109) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(110) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(100) Filter +(111) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(101) Exchange +(112) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(103) Project +(114) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(115) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(118) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(109) Project +(120) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(121) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(113) Exchange +(124) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(115) Project +(126) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(116) Exchange +(127) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(119) Exchange +(130) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(121) Project +(132) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(122) Exchange +(133) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(125) Exchange +(136) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(137) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(127) Project +(138) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(128) HashAggregate +(139) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(129) Exchange +(140) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) HashAggregate +(141) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(131) Exchange +(142) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(143) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(133) AdaptiveSparkPlan +(144) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt index 63b30bb5d26b..5ad7bc9d3675 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt index 6213f61ffdf4..f83e7572840a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt @@ -1,78 +1,84 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- TakeOrderedAndProjectExecTransformer (59) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ShuffleQueryStage (36), Statistics(X) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ShuffleQueryStage (21), Statistics(X) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ShuffleQueryStage (29), Statistics(X) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ NoopFilter (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (66) + +- TakeOrderedAndProjectExecTransformer (65) + +- ^ ProjectExecTransformer (63) + +- ^ RegularHashAggregateExecTransformer (62) + +- ^ InputIteratorTransformer (61) + +- ShuffleQueryStage (59), Statistics(X) + +- ColumnarExchange (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24), Statistics(X) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ NoopFilter (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33), Statistics(X) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ NoopFilter (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ NoopFilter (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- ShuffledHashJoin Inner BuildRight (81) - :- Exchange (77) - : +- Project (76) - : +- ShuffledHashJoin Inner BuildRight (75) - : :- Exchange (70) - : : +- Project (69) - : : +- ShuffledHashJoin Inner BuildRight (68) - : : :- Exchange (63) - : : : +- Filter (62) - : : : +- Scan parquet (61) - : : +- Exchange (67) - : : +- Project (66) - : : +- Filter (65) - : : +- Scan parquet (64) - : +- Exchange (74) - : +- Project (73) - : +- Filter (72) - : +- Scan parquet (71) - +- Exchange (80) - +- Filter (79) - +- Scan parquet (78) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- ShuffledHashJoin Inner BuildRight (87) + :- Exchange (83) + : +- Project (82) + : +- ShuffledHashJoin Inner BuildRight (81) + : :- Exchange (76) + : : +- Project (75) + : : +- ShuffledHashJoin Inner BuildRight (74) + : : :- Exchange (69) + : : : +- Filter (68) + : : : +- Scan parquet (67) + : : +- Exchange (73) + : : +- Project (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (80) + : +- Project (79) + : +- Filter (78) + : +- Scan parquet (77) + +- Exchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -94,358 +100,382 @@ Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acct Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: X + +(6) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(55) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(56) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(58) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(59) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(60) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(61) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(62) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(57) ProjectExecTransformer +(63) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(58) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(59) TakeOrderedAndProjectExecTransformer +(65) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(60) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(61) Scan parquet +(67) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) Filter +(68) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(63) Exchange +(69) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(65) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(66) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(67) Exchange +(73) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) ShuffledHashJoin +(74) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(69) Project +(75) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(70) Exchange +(76) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(77) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(72) Filter +(78) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(73) Project +(79) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(74) Exchange +(80) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(81) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(76) Project +(82) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(77) Exchange +(83) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(80) Exchange +(86) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(87) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(88) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(89) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(84) Exchange +(90) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(91) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(86) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt index 80dbdb2c50ad..b3fb729d8d96 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt @@ -1,65 +1,70 @@ == Physical Plan == -AdaptiveSparkPlan (72) +AdaptiveSparkPlan (77) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ SortExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ShuffleQueryStage (38), Statistics(X) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (55) + +- ^ SortExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ FilterExecTransformer (46) + +- ^ RegularHashAggregateExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42), Statistics(X) + +- ColumnarExchange (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ NoopFilter (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (71) - +- Exchange (70) - +- Filter (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- ShuffledHashJoin Inner BuildRight (64) - :- Exchange (59) - : +- Project (58) - : +- ShuffledHashJoin Inner BuildRight (57) - : :- Exchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- Exchange (56) - : +- Filter (55) - : +- Scan parquet (54) - +- Exchange (63) - +- Project (62) - +- Filter (61) - +- Scan parquet (60) + Sort (76) + +- Exchange (75) + +- Filter (74) + +- HashAggregate (73) + +- Exchange (72) + +- HashAggregate (71) + +- Project (70) + +- ShuffledHashJoin Inner BuildRight (69) + :- Exchange (64) + : +- Project (63) + : +- ShuffledHashJoin Inner BuildRight (62) + : :- Exchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- Exchange (61) + : +- Filter (60) + : +- Scan parquet (59) + +- Exchange (68) + +- Project (67) + +- Filter (66) + +- Scan parquet (65) (1) Scan parquet @@ -81,549 +86,579 @@ Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [1]: [n_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [n_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(34) FlushableHashAggregateExecTransformer +(38) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(41) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(43) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(45) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(42) FilterExecTransformer +(46) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(43) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(44) ColumnarExchange +(48) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(49) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(46) InputAdapter +(51) InputAdapter Input [2]: [ps_partkey#X, value#X] -(47) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(48) SortExecTransformer +(53) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(49) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(50) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(51) Scan parquet +(56) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(52) Filter +(57) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(53) Exchange +(58) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(59) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(60) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) Exchange +(61) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(62) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(58) Project +(63) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(59) Exchange +(64) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(65) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(61) Filter +(66) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(62) Project +(67) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(63) Exchange +(68) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(69) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(65) Project +(70) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(66) HashAggregate +(71) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(67) Exchange +(72) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(73) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(69) Filter +(74) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(70) Exchange +(75) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(76) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(72) AdaptiveSparkPlan +(77) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (120) +Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (127) +- == Final Plan == - VeloxColumnarToRowExec (102) - +- ^ ProjectExecTransformer (100) - +- ^ RegularHashAggregateExecTransformer (99) - +- ^ RegularHashAggregateExecTransformer (98) - +- ^ ProjectExecTransformer (97) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) - :- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) - : :- ^ InputIteratorTransformer (80) - : : +- ShuffleQueryStage (78), Statistics(X) - : : +- ColumnarExchange (77) - : : +- ^ ProjectExecTransformer (75) - : : +- ^ NoopFilter (74) - : : +- ^ Scan parquet (73) - : +- ^ InputIteratorTransformer (84) - : +- ShuffleQueryStage (82), Statistics(X) - : +- ReusedExchange (81) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ReusedExchange (92) + VeloxColumnarToRowExec (109) + +- ^ ProjectExecTransformer (107) + +- ^ RegularHashAggregateExecTransformer (106) + +- ^ RegularHashAggregateExecTransformer (105) + +- ^ ProjectExecTransformer (104) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (103) + :- ^ InputIteratorTransformer (98) + : +- ShuffleQueryStage (96), Statistics(X) + : +- ColumnarExchange (95) + : +- VeloxAppendBatches (94) + : +- ^ ProjectExecTransformer (92) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (91) + : :- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ NoopFilter (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (90) + : +- ShuffleQueryStage (88), Statistics(X) + : +- ReusedExchange (87) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ReusedExchange (99) +- == Initial Plan == - HashAggregate (119) - +- HashAggregate (118) - +- Project (117) - +- ShuffledHashJoin Inner BuildRight (116) - :- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildRight (109) - : :- Exchange (105) - : : +- Filter (104) - : : +- Scan parquet (103) - : +- Exchange (108) - : +- Filter (107) - : +- Scan parquet (106) - +- Exchange (115) - +- Project (114) - +- Filter (113) - +- Scan parquet (112) - - -(73) Scan parquet + HashAggregate (126) + +- HashAggregate (125) + +- Project (124) + +- ShuffledHashJoin Inner BuildRight (123) + :- Exchange (118) + : +- Project (117) + : +- ShuffledHashJoin Inner BuildRight (116) + : :- Exchange (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- Exchange (115) + : +- Filter (114) + : +- Scan parquet (113) + +- Exchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) + + +(78) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(74) NoopFilter +(79) NoopFilter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(75) ProjectExecTransformer +(80) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(76) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(77) ColumnarExchange +(82) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(83) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(78) ShuffleQueryStage +(84) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(79) InputAdapter +(85) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(80) InputIteratorTransformer +(86) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(81) ReusedExchange [Reuses operator id: 13] +(87) ReusedExchange [Reuses operator id: 15] Output [2]: [s_suppkey#X, s_nationkey#X] -(82) ShuffleQueryStage +(88) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(83) InputAdapter +(89) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(84) InputIteratorTransformer +(90) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(85) ShuffledHashJoinExecTransformer +(91) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(86) ProjectExecTransformer +(92) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(87) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(88) ColumnarExchange +(94) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(95) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(96) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(90) InputAdapter +(97) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(91) InputIteratorTransformer +(98) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(92) ReusedExchange [Reuses operator id: 28] +(99) ReusedExchange [Reuses operator id: 32] Output [1]: [n_nationkey#X] -(93) ShuffleQueryStage +(100) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(94) InputAdapter +(101) InputAdapter Input [1]: [n_nationkey#X] -(95) InputIteratorTransformer +(102) InputIteratorTransformer Input [1]: [n_nationkey#X] -(96) ShuffledHashJoinExecTransformer +(103) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(97) ProjectExecTransformer +(104) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(98) RegularHashAggregateExecTransformer +(105) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(99) RegularHashAggregateExecTransformer +(106) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(100) ProjectExecTransformer +(107) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(101) WholeStageCodegenTransformer (X) +(108) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(102) VeloxColumnarToRowExec +(109) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(103) Scan parquet +(110) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(104) Filter +(111) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(105) Exchange +(112) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(113) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(107) Filter +(114) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(108) Exchange +(115) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(116) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(110) Project +(117) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(111) Exchange +(118) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(119) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(113) Filter +(120) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(114) Project +(121) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(115) Exchange +(122) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(123) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(117) Project +(124) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(118) HashAggregate +(125) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(119) HashAggregate +(126) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(120) AdaptiveSparkPlan +(127) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt index 58a8788ca252..c9a5452d9aa0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt @@ -1,46 +1,49 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin Inner BuildLeft (42) - :- Exchange (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- ShuffledHashJoin Inner BuildLeft (45) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -62,196 +65,208 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(31) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) Scan parquet +(38) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(36) Filter +(39) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(37) Exchange +(40) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(41) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(40) Project +(43) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(41) Exchange +(44) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(45) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(43) Project +(46) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(44) HashAggregate +(47) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(45) Exchange +(48) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(49) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(47) Exchange +(50) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(51) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt index 8837c6ef4143..0a402001c23b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt @@ -1,49 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftOuter BuildRight (43) - :- Exchange (38) - : +- Scan parquet (37) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftOuter BuildRight (46) + :- Exchange (41) + : +- Scan parquet (40) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -60,223 +63,235 @@ Input [1]: [c_custkey#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(4) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) NoopFilter +(10) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(44) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) Exchange +(45) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(47) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(48) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt index 2c06906d4fb8..cb3ddbb3a2f7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (23) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (25) + +- ^ ProjectExecTransformer (23) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (34) - +- HashAggregate (33) - +- Project (32) - +- ShuffledHashJoin Inner BuildRight (31) - :- Exchange (27) - : +- Project (26) - : +- Filter (25) - : +- Scan parquet (24) - +- Exchange (30) - +- Filter (29) - +- Scan parquet (28) + HashAggregate (36) + +- HashAggregate (35) + +- Project (34) + +- ShuffledHashJoin Inner BuildRight (33) + :- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -52,144 +54,152 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [2]: [p_partkey#X, p_type#X] Arguments: [p_partkey#X, p_type#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(21) ProjectExecTransformer +(23) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(22) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(23) VeloxColumnarToRowExec +(25) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(24) Scan parquet +(26) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(26) Project +(28) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) Exchange +(29) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Scan parquet +(30) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(30) Exchange +(32) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) ShuffledHashJoin +(33) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(32) Project +(34) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(34) HashAggregate +(36) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] -(35) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt index 270cea0f6f46..7f0635c0d9a4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt @@ -1,43 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- AQEShuffleRead (26) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (29) + +- AQEShuffleRead (28) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (22) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- Project (39) - +- ShuffledHashJoin Inner BuildLeft (38) - :- Exchange (30) - : +- Filter (29) - : +- Scan parquet (28) - +- Filter (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (43) + +- Exchange (42) + +- Project (41) + +- ShuffledHashJoin Inner BuildLeft (40) + :- Exchange (32) + : +- Filter (31) + : +- Scan parquet (30) + +- Filter (39) + +- HashAggregate (38) + +- Exchange (37) + +- HashAggregate (36) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -59,328 +61,336 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(16) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(20) FilterExecTransformer +(21) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(21) ShuffledHashJoinExecTransformer +(22) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(26) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) AQEShuffleRead +(28) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(27) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) Scan parquet +(30) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(30) Exchange +(32) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) Scan parquet +(33) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(32) Filter +(34) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(33) Project +(35) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(34) HashAggregate +(36) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(35) Exchange +(37) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(38) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(37) Filter +(39) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(38) ShuffledHashJoin +(40) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(39) Project +(41) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(40) Exchange +(42) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(43) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (67) +Subquery:1 Hosting operator id = 21 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ RegularHashAggregateExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ ProjectExecTransformer (47) - +- ^ FlushableHashAggregateExecTransformer (46) - +- ^ ProjectExecTransformer (45) - +- ^ NoopFilter (44) - +- ^ Scan parquet (43) + VeloxColumnarToRowExec (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ ProjectExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ InputIteratorTransformer (54) + +- ShuffleQueryStage (52), Statistics(X) + +- ColumnarExchange (51) + +- ^ ProjectExecTransformer (49) + +- ^ FlushableHashAggregateExecTransformer (48) + +- ^ ProjectExecTransformer (47) + +- ^ NoopFilter (46) + +- ^ Scan parquet (45) +- == Initial Plan == - HashAggregate (66) - +- HashAggregate (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (68) + +- HashAggregate (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- Filter (62) + +- Scan parquet (61) -(43) Scan parquet +(45) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(44) NoopFilter +(46) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(45) ProjectExecTransformer +(47) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) FlushableHashAggregateExecTransformer +(48) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(48) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(49) ColumnarExchange +(51) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(51) InputAdapter +(53) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(53) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(54) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(55) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(56) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(57) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(58) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(59) Scan parquet +(61) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(60) Filter +(62) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(61) Project +(63) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(63) Exchange +(65) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(65) HashAggregate +(67) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(66) HashAggregate +(68) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(67) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt index 90c77f0de9b3..b9a3fb8dadd0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt @@ -1,59 +1,62 @@ == Physical Plan == -AdaptiveSparkPlan (64) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (42) - +- ^ SortExecTransformer (40) - +- ^ InputIteratorTransformer (39) - +- ShuffleQueryStage (37), Statistics(X) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (45) + +- ^ SortExecTransformer (43) + +- ^ InputIteratorTransformer (42) + +- ShuffleQueryStage (40), Statistics(X) + +- ColumnarExchange (39) + +- VeloxAppendBatches (38) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- ^ ProjectExecTransformer (30) + +- ^ FlushableHashAggregateExecTransformer (29) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (63) - +- Exchange (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- HashAggregate (58) - +- Exchange (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (50) - : +- BroadcastHashJoin LeftAnti BuildRight (49) - : :- Filter (44) - : : +- Scan parquet (43) - : +- BroadcastExchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (53) - +- Filter (52) - +- Scan parquet (51) + Sort (66) + +- Exchange (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- HashAggregate (61) + +- Exchange (60) + +- HashAggregate (59) + +- Project (58) + +- ShuffledHashJoin Inner BuildRight (57) + :- Exchange (53) + : +- BroadcastHashJoin LeftAnti BuildRight (52) + : :- Filter (47) + : : +- Scan parquet (46) + : +- BroadcastExchange (51) + : +- Project (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -75,270 +78,282 @@ Input [2]: [ps_partkey#X, ps_suppkey#X] Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) FlushableHashAggregateExecTransformer +(29) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(30) ColumnarExchange +(32) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(32) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) ColumnarExchange +(38) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(39) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(40) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(38) InputAdapter +(41) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) InputIteratorTransformer +(42) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(40) SortExecTransformer +(43) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(41) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(42) VeloxColumnarToRowExec +(45) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) Scan parquet +(46) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(45) Scan parquet +(48) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(46) Filter +(49) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(47) Project +(50) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(48) BroadcastExchange +(51) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(49) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(50) Exchange +(53) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(54) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(53) Exchange +(56) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(57) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(55) Project +(58) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(56) HashAggregate +(59) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) HashAggregate +(61) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(59) HashAggregate +(62) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(60) Exchange +(63) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(62) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) Sort +(66) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(64) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt index 16321ffa4e59..6206907a8588 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt @@ -1,56 +1,58 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ ProjectExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) - :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ FilterExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ NoopFilter (20) - +- ^ Scan parquet (19) + VeloxColumnarToRowExec (39) + +- ^ ProjectExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ ProjectExecTransformer (34) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (33) + :- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ NoopFilter (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (56) - +- HashAggregate (55) - +- Project (54) - +- ShuffledHashJoin Inner BuildRight (53) - :- Project (46) - : +- ShuffledHashJoin Inner BuildRight (45) - : :- Exchange (40) - : : +- Filter (39) - : : +- Scan parquet (38) - : +- Exchange (44) - : +- Project (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Filter (48) - +- Scan parquet (47) + HashAggregate (58) + +- HashAggregate (57) + +- Project (56) + +- ShuffledHashJoin Inner BuildRight (55) + :- Project (48) + : +- ShuffledHashJoin Inner BuildRight (47) + : :- Exchange (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Exchange (46) + : +- Project (45) + : +- Filter (44) + : +- Scan parquet (43) + +- Filter (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Filter (50) + +- Scan parquet (49) (1) Scan parquet @@ -72,250 +74,258 @@ Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_container#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(19) Scan parquet +(21) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(20) NoopFilter +(22) NoopFilter Input [2]: [l_partkey#X, l_quantity#X] Arguments: [l_partkey#X, l_quantity#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(30) FilterExecTransformer +(32) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(31) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(32) ProjectExecTransformer +(34) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(33) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(35) ProjectExecTransformer +(37) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(36) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(37) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(38) Scan parquet +(40) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(39) Filter +(41) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(40) Exchange +(42) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(43) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(43) Project +(45) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(44) Exchange +(46) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(46) Project +(48) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(47) Scan parquet +(49) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(48) Filter +(50) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(49) HashAggregate +(51) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(50) Exchange +(52) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) Filter +(54) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(53) ShuffledHashJoin +(55) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(54) Project +(56) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) HashAggregate +(57) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(56) HashAggregate +(58) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] -(57) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt index fa441bac82cc..038422e0fecf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt @@ -1,89 +1,94 @@ == Physical Plan == -AdaptiveSparkPlan (97) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- TakeOrderedAndProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) - :- ^ InputIteratorTransformer (41) - : +- ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ShuffleQueryStage (22), Statistics(X) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) - :- ^ InputIteratorTransformer (49) - : +- ShuffleQueryStage (47), Statistics(X) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ NoopFilter (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ShuffleQueryStage (51), Statistics(X) - +- ReusedExchange (50) + VeloxColumnarToRowExec (69) + +- TakeOrderedAndProjectExecTransformer (68) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ RegularHashAggregateExecTransformer (65) + +- ^ ProjectExecTransformer (64) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (63) + :- ^ InputIteratorTransformer (45) + : +- ShuffleQueryStage (43), Statistics(X) + : +- ColumnarExchange (42) + : +- VeloxAppendBatches (41) + : +- ^ ProjectExecTransformer (39) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (38) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (37) + : +- ShuffleQueryStage (35), Statistics(X) + : +- ColumnarExchange (34) + : +- VeloxAppendBatches (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (30) + : :- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ NoopFilter (11) + : : +- ^ Scan parquet (10) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ RegularHashAggregateExecTransformer (27) + : +- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) + :- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- VeloxAppendBatches (50) + : +- ^ ProjectExecTransformer (48) + : +- ^ NoopFilter (47) + : +- ^ Scan parquet (46) + +- ^ ProjectExecTransformer (61) + +- ^ FilterExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ReusedExchange (55) +- == Initial Plan == - TakeOrderedAndProject (96) - +- HashAggregate (95) - +- HashAggregate (94) - +- Project (93) - +- ShuffledHashJoin Inner BuildRight (92) - :- Exchange (81) - : +- Project (80) - : +- ShuffledHashJoin Inner BuildLeft (79) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (78) - : +- ShuffledHashJoin LeftSemi BuildRight (77) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Project (76) - : +- Filter (75) - : +- HashAggregate (74) - : +- Exchange (73) - : +- HashAggregate (72) - : +- Scan parquet (71) - +- ShuffledHashJoin LeftSemi BuildRight (91) - :- Exchange (84) - : +- Filter (83) - : +- Scan parquet (82) - +- Project (90) - +- Filter (89) - +- HashAggregate (88) - +- Exchange (87) - +- HashAggregate (86) - +- Scan parquet (85) + TakeOrderedAndProject (101) + +- HashAggregate (100) + +- HashAggregate (99) + +- Project (98) + +- ShuffledHashJoin Inner BuildRight (97) + :- Exchange (86) + : +- Project (85) + : +- ShuffledHashJoin Inner BuildLeft (84) + : :- Exchange (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (83) + : +- ShuffledHashJoin LeftSemi BuildRight (82) + : :- Exchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- Project (81) + : +- Filter (80) + : +- HashAggregate (79) + : +- Exchange (78) + : +- HashAggregate (77) + : +- Scan parquet (76) + +- ShuffledHashJoin LeftSemi BuildRight (96) + :- Exchange (89) + : +- Filter (88) + : +- Scan parquet (87) + +- Project (95) + +- Filter (94) + +- HashAggregate (93) + +- Exchange (92) + +- HashAggregate (91) + +- Scan parquet (90) (1) Scan parquet @@ -105,420 +110,440 @@ Input [2]: [c_custkey#X, c_name#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(26) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(27) ProjectExecTransformer +(29) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(28) ShuffledHashJoinExecTransformer +(30) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(30) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(31) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(33) InputAdapter +(36) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(34) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) ShuffledHashJoinExecTransformer +(38) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(36) ProjectExecTransformer +(39) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(38) ColumnarExchange +(41) VeloxAppendBatches +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(42) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(43) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(40) InputAdapter +(44) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(41) InputIteratorTransformer +(45) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(42) Scan parquet +(46) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(43) NoopFilter +(47) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(46) ColumnarExchange +(50) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: X + +(51) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(52) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(48) InputAdapter +(53) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(49) InputIteratorTransformer +(54) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(50) ReusedExchange [Reuses operator id: 21] +(55) ReusedExchange [Reuses operator id: 23] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) ShuffleQueryStage +(56) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(52) InputAdapter +(57) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(53) InputIteratorTransformer +(58) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(54) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(55) FilterExecTransformer +(60) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(56) ProjectExecTransformer +(61) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(58) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(59) ProjectExecTransformer +(64) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(60) RegularHashAggregateExecTransformer +(65) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(61) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(62) WholeStageCodegenTransformer (X) +(67) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(63) TakeOrderedAndProjectExecTransformer +(68) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(64) VeloxColumnarToRowExec +(69) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(65) Scan parquet +(70) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(66) Filter +(71) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(67) Exchange +(72) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(73) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(69) Filter +(74) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(70) Exchange +(75) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(76) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(77) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(78) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(79) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(80) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(81) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Exchange +(83) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) ShuffledHashJoin +(84) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(80) Project +(85) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(81) Exchange +(86) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(87) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(83) Filter +(88) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(84) Exchange +(89) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(90) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(86) HashAggregate +(91) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(87) Exchange +(92) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) HashAggregate +(93) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(89) Filter +(94) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(90) Project +(95) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(91) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(93) Project +(98) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(94) HashAggregate +(99) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(95) HashAggregate +(100) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(96) TakeOrderedAndProject +(101) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(97) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt index d9ceedf0db6c..440383aa1cd7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (33) - +- HashAggregate (32) - +- Project (31) - +- ShuffledHashJoin Inner BuildRight (30) - :- Exchange (26) - : +- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- Exchange (29) - +- Filter (28) - +- Scan parquet (27) + HashAggregate (35) + +- HashAggregate (34) + +- Project (33) + +- ShuffledHashJoin Inner BuildRight (32) + :- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) (1) Scan parquet @@ -51,140 +53,148 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(24) VeloxColumnarToRowExec Input [1]: [revenue#X] -(23) Scan parquet +(25) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(24) Filter +(26) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(25) Project +(27) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(26) Exchange +(28) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Scan parquet +(29) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(28) Filter +(30) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(29) Exchange +(31) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) ShuffledHashJoin +(32) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(31) Project +(33) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(32) HashAggregate +(34) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(34) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt index 305f98339a44..d3cff30e0ce2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt @@ -1,109 +1,119 @@ == Physical Plan == -AdaptiveSparkPlan (123) +AdaptiveSparkPlan (133) +- == Final Plan == - VeloxColumnarToRowExec (83) - +- AQEShuffleRead (82) - +- ShuffleQueryStage (81), Statistics(X) - +- ColumnarExchange (80) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ShuffleQueryStage (29), Statistics(X) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ShuffleQueryStage (14), Statistics(X) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ NoopFilter (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ShuffleQueryStage (22), Statistics(X) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ NoopFilter (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ NoopFilter (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41), Statistics(X) - : +- ReusedExchange (40) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (93) + +- AQEShuffleRead (92) + +- ShuffleQueryStage (91), Statistics(X) + +- ColumnarExchange (90) + +- VeloxAppendBatches (89) + +- ^ ProjectExecTransformer (87) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (86) + :- ^ InputIteratorTransformer (76) + : +- ShuffleQueryStage (74), Statistics(X) + : +- ColumnarExchange (73) + : +- VeloxAppendBatches (72) + : +- ^ ProjectExecTransformer (70) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (69) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- VeloxAppendBatches (64) + : +- ^ ProjectExecTransformer (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (61) + : :- ^ InputIteratorTransformer (35) + : : +- ShuffleQueryStage (33), Statistics(X) + : : +- ColumnarExchange (32) + : : +- VeloxAppendBatches (31) + : : +- ^ ProjectExecTransformer (29) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : : :- ^ InputIteratorTransformer (18) + : : : +- ShuffleQueryStage (16), Statistics(X) + : : : +- ColumnarExchange (15) + : : : +- VeloxAppendBatches (14) + : : : +- ^ ProjectExecTransformer (12) + : : : +- ^ NoopFilter (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (27) + : : +- ShuffleQueryStage (25), Statistics(X) + : : +- ColumnarExchange (24) + : : +- VeloxAppendBatches (23) + : : +- ^ ProjectExecTransformer (21) + : : +- ^ NoopFilter (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (60) + : +- ShuffleQueryStage (58), Statistics(X) + : +- ColumnarExchange (57) + : +- VeloxAppendBatches (56) + : +- ^ ProjectExecTransformer (54) + : +- ^ FilterExecTransformer (53) + : +- ^ ProjectExecTransformer (52) + : +- ^ RegularHashAggregateExecTransformer (51) + : +- ^ RegularHashAggregateExecTransformer (50) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (49) + : :- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ NoopFilter (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (48) + : +- ShuffleQueryStage (46), Statistics(X) + : +- ReusedExchange (45) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- VeloxAppendBatches (81) + +- ^ ProjectExecTransformer (79) + +- ^ NoopFilter (78) + +- ^ Scan parquet (77) +- == Initial Plan == - Sort (122) - +- Exchange (121) - +- Project (120) - +- ShuffledHashJoin Inner BuildRight (119) - :- Exchange (114) - : +- Project (113) - : +- ShuffledHashJoin LeftSemi BuildRight (112) - : :- Exchange (86) - : : +- Filter (85) - : : +- Scan parquet (84) - : +- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildLeft (109) - : :- Exchange (95) - : : +- ShuffledHashJoin LeftSemi BuildRight (94) - : : :- Exchange (89) - : : : +- Filter (88) - : : : +- Scan parquet (87) - : : +- Exchange (93) - : : +- Project (92) - : : +- Filter (91) - : : +- Scan parquet (90) - : +- Exchange (108) - : +- Filter (107) - : +- HashAggregate (106) - : +- HashAggregate (105) - : +- ShuffledHashJoin LeftSemi BuildRight (104) - : :- Exchange (99) - : : +- Project (98) - : : +- Filter (97) - : : +- Scan parquet (96) - : +- Exchange (103) - : +- Project (102) - : +- Filter (101) - : +- Scan parquet (100) - +- Exchange (118) - +- Project (117) - +- Filter (116) - +- Scan parquet (115) + Sort (132) + +- Exchange (131) + +- Project (130) + +- ShuffledHashJoin Inner BuildRight (129) + :- Exchange (124) + : +- Project (123) + : +- ShuffledHashJoin LeftSemi BuildRight (122) + : :- Exchange (96) + : : +- Filter (95) + : : +- Scan parquet (94) + : +- Exchange (121) + : +- Project (120) + : +- ShuffledHashJoin Inner BuildLeft (119) + : :- Exchange (105) + : : +- ShuffledHashJoin LeftSemi BuildRight (104) + : : :- Exchange (99) + : : : +- Filter (98) + : : : +- Scan parquet (97) + : : +- Exchange (103) + : : +- Project (102) + : : +- Filter (101) + : : +- Scan parquet (100) + : +- Exchange (118) + : +- Filter (117) + : +- HashAggregate (116) + : +- HashAggregate (115) + : +- ShuffledHashJoin LeftSemi BuildRight (114) + : :- Exchange (109) + : : +- Project (108) + : : +- Filter (107) + : : +- Scan parquet (106) + : +- Exchange (113) + : +- Project (112) + : +- Filter (111) + : +- Scan parquet (110) + +- Exchange (128) + +- Project (127) + +- Filter (126) + +- Scan parquet (125) (1) Scan parquet @@ -125,508 +135,548 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(18) NoopFilter +(20) NoopFilter Input [2]: [p_partkey#X, p_name#X] Arguments: [p_partkey#X, p_name#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [1]: [p_partkey#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [1]: [p_partkey#X] -(25) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(32) Scan parquet +(36) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(33) NoopFilter +(37) NoopFilter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(40) ReusedExchange [Reuses operator id: 21] +(45) ReusedExchange [Reuses operator id: 24] Output [1]: [p_partkey#X] -(41) ShuffleQueryStage +(46) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(42) InputAdapter +(47) InputAdapter Input [1]: [p_partkey#X] -(43) InputIteratorTransformer +(48) InputIteratorTransformer Input [1]: [p_partkey#X] -(44) ShuffledHashJoinExecTransformer +(49) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(45) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(46) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(47) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(48) FilterExecTransformer +(53) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(49) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(51) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(53) InputAdapter +(59) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(54) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(55) ShuffledHashJoinExecTransformer +(61) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(58) ColumnarExchange +(64) VeloxAppendBatches +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: X + +(65) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(66) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(60) InputAdapter +(67) InputAdapter Input [1]: [ps_suppkey#X] -(61) InputIteratorTransformer +(68) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(62) ShuffledHashJoinExecTransformer +(69) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(63) ProjectExecTransformer +(70) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(71) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(72) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(73) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(74) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(75) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(68) InputIteratorTransformer +(76) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(69) Scan parquet +(77) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(70) NoopFilter +(78) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(71) ProjectExecTransformer +(79) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(73) ColumnarExchange +(81) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(82) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(83) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(75) InputAdapter +(84) InputAdapter Input [1]: [n_nationkey#X] -(76) InputIteratorTransformer +(85) InputIteratorTransformer Input [1]: [n_nationkey#X] -(77) ShuffledHashJoinExecTransformer +(86) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(87) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(79) WholeStageCodegenTransformer (X) +(88) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(80) ColumnarExchange +(89) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(90) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(91) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(82) AQEShuffleRead +(92) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(83) VeloxColumnarToRowExec +(93) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(84) Scan parquet +(94) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(85) Filter +(95) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(86) Exchange +(96) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Scan parquet +(97) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(88) Filter +(98) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(89) Exchange +(99) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(100) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(91) Filter +(101) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(92) Project +(102) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(93) Exchange +(103) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) ShuffledHashJoin +(104) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(95) Exchange +(105) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) Scan parquet +(106) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(97) Filter +(107) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(98) Project +(108) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(102) Project +(112) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(103) Exchange +(113) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(105) HashAggregate +(115) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(106) HashAggregate +(116) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(107) Filter +(117) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(108) Exchange +(118) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(110) Project +(120) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(111) Exchange +(121) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(113) Project +(123) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(114) Exchange +(124) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) Scan parquet +(125) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(116) Filter +(126) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(117) Project +(127) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(118) Exchange +(128) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(129) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(120) Project +(130) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(121) Exchange +(131) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Sort +(132) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(123) AdaptiveSparkPlan +(133) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt index 04cc1f47a3d3..d2c18146af4f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt @@ -1,103 +1,112 @@ == Physical Plan == -AdaptiveSparkPlan (118) +AdaptiveSparkPlan (127) +- == Final Plan == - VeloxColumnarToRowExec (81) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ShuffleQueryStage (21), Statistics(X) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ShuffleQueryStage (30), Statistics(X) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ NoopFilter (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ NoopFilter (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ShuffleQueryStage (67), Statistics(X) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ NoopFilter (63) - +- ^ Scan parquet (62) + VeloxColumnarToRowExec (90) + +- ^ RegularHashAggregateExecTransformer (88) + +- ^ InputIteratorTransformer (87) + +- ShuffleQueryStage (85), Statistics(X) + +- ColumnarExchange (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24), Statistics(X) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34), Statistics(X) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ NoopFilter (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ NoopFilter (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ NoopFilter (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (117) - +- HashAggregate (116) - +- Exchange (115) - +- HashAggregate (114) - +- Project (113) - +- ShuffledHashJoin Inner BuildRight (112) - :- Exchange (107) - : +- Project (106) - : +- ShuffledHashJoin Inner BuildRight (105) - : :- Exchange (100) - : : +- Project (99) - : : +- ShuffledHashJoin Inner BuildLeft (98) - : : :- Exchange (84) - : : : +- Filter (83) - : : : +- Scan parquet (82) - : : +- Exchange (97) - : : +- ShuffledHashJoin LeftAnti BuildRight (96) - : : :- ShuffledHashJoin LeftSemi BuildRight (91) - : : : :- Exchange (88) - : : : : +- Project (87) - : : : : +- Filter (86) - : : : : +- Scan parquet (85) - : : : +- Exchange (90) - : : : +- Scan parquet (89) - : : +- Exchange (95) - : : +- Project (94) - : : +- Filter (93) - : : +- Scan parquet (92) - : +- Exchange (104) - : +- Project (103) - : +- Filter (102) - : +- Scan parquet (101) - +- Exchange (111) - +- Project (110) - +- Filter (109) - +- Scan parquet (108) + TakeOrderedAndProject (126) + +- HashAggregate (125) + +- Exchange (124) + +- HashAggregate (123) + +- Project (122) + +- ShuffledHashJoin Inner BuildRight (121) + :- Exchange (116) + : +- Project (115) + : +- ShuffledHashJoin Inner BuildRight (114) + : :- Exchange (109) + : : +- Project (108) + : : +- ShuffledHashJoin Inner BuildLeft (107) + : : :- Exchange (93) + : : : +- Filter (92) + : : : +- Scan parquet (91) + : : +- Exchange (106) + : : +- ShuffledHashJoin LeftAnti BuildRight (105) + : : :- ShuffledHashJoin LeftSemi BuildRight (100) + : : : :- Exchange (97) + : : : : +- Project (96) + : : : : +- Filter (95) + : : : : +- Scan parquet (94) + : : : +- Exchange (99) + : : : +- Scan parquet (98) + : : +- Exchange (104) + : : +- Project (103) + : : +- Filter (102) + : : +- Scan parquet (101) + : +- Exchange (113) + : +- Project (112) + : +- Filter (111) + : +- Scan parquet (110) + +- Exchange (120) + +- Project (119) + +- Filter (118) + +- Scan parquet (117) (1) Scan parquet @@ -119,490 +128,526 @@ Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(24) ShuffledHashJoinExecTransformer +(27) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(25) Scan parquet +(28) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(26) NoopFilter +(29) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(29) ColumnarExchange +(32) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(33) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(33) ShuffledHashJoinExecTransformer +(37) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(41) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(40) ShuffledHashJoinExecTransformer +(45) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(47) Scan parquet +(53) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(48) NoopFilter +(54) NoopFilter Input [2]: [o_orderkey#X, o_orderstatus#X] Arguments: [o_orderkey#X, o_orderstatus#X] -(49) ProjectExecTransformer +(55) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(50) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(51) ColumnarExchange +(57) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: X + +(58) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(53) InputAdapter +(60) InputAdapter Input [1]: [o_orderkey#X] -(54) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [o_orderkey#X] -(55) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(62) Scan parquet +(70) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(63) NoopFilter +(71) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(64) ProjectExecTransformer +(72) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(65) WholeStageCodegenTransformer (X) +(73) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(66) ColumnarExchange +(74) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(75) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(76) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(68) InputAdapter +(77) InputAdapter Input [1]: [n_nationkey#X] -(69) InputIteratorTransformer +(78) InputIteratorTransformer Input [1]: [n_nationkey#X] -(70) ShuffledHashJoinExecTransformer +(79) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(72) FlushableHashAggregateExecTransformer +(81) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(73) ProjectExecTransformer +(82) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(74) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(75) ColumnarExchange +(84) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(85) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(77) InputAdapter +(86) InputAdapter Input [2]: [s_name#X, count#X] -(78) InputIteratorTransformer +(87) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(79) RegularHashAggregateExecTransformer +(88) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(80) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(81) VeloxColumnarToRowExec +(90) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(82) Scan parquet +(91) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(83) Filter +(92) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(84) Exchange +(93) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(94) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(86) Filter +(95) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(87) Project +(96) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(88) Exchange +(97) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) Scan parquet +(98) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(90) Exchange +(99) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) ShuffledHashJoin +(100) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(92) Scan parquet +(101) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(93) Filter +(102) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(94) Project +(103) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(95) Exchange +(104) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(105) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(97) Exchange +(106) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(99) Project +(108) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(100) Exchange +(109) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Scan parquet +(110) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(102) Filter +(111) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(103) Project +(112) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(104) Exchange +(113) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(106) Project +(115) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(107) Exchange +(116) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Scan parquet +(117) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(109) Filter +(118) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(110) Project +(119) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(111) Exchange +(120) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(121) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(113) Project +(122) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(114) HashAggregate +(123) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(115) Exchange +(124) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) HashAggregate +(125) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(117) TakeOrderedAndProject +(126) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(118) AdaptiveSparkPlan +(127) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt index 9a513e223197..ea3839718ac1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt @@ -1,43 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (46) +AdaptiveSparkPlan (49) +- == Final Plan == - VeloxColumnarToRowExec (33) - +- ^ SortExecTransformer (31) - +- ^ InputIteratorTransformer (30) - +- ShuffleQueryStage (28), Statistics(X) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (45) - +- Exchange (44) - +- HashAggregate (43) - +- Exchange (42) - +- HashAggregate (41) - +- Project (40) - +- ShuffledHashJoin LeftAnti BuildRight (39) - :- Exchange (36) - : +- Filter (35) - : +- Scan parquet (34) - +- Exchange (38) - +- Scan parquet (37) + Sort (48) + +- Exchange (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- ShuffledHashJoin LeftAnti BuildRight (42) + :- Exchange (39) + : +- Filter (38) + : +- Scan parquet (37) + +- Exchange (41) + +- Scan parquet (40) (1) Scan parquet @@ -59,296 +62,308 @@ Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) Scan parquet +(10) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [1]: [o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [1]: [o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(26) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) ColumnarExchange +(29) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(30) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(28) ShuffleQueryStage +(31) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(29) InputAdapter +(32) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) InputIteratorTransformer +(33) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(31) SortExecTransformer +(34) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(32) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(33) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) Scan parquet +(37) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(35) Filter +(38) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(36) Exchange +(39) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(40) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(40) Project +(43) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(41) HashAggregate +(44) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(42) Exchange +(45) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) HashAggregate +(46) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(44) Exchange +(47) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(48) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(46) AdaptiveSparkPlan +(49) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (65) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ NoopFilter (48) - +- ^ Scan parquet (47) + VeloxColumnarToRowExec (61) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- ^ FlushableHashAggregateExecTransformer (53) + +- ^ ProjectExecTransformer (52) + +- ^ NoopFilter (51) + +- ^ Scan parquet (50) +- == Initial Plan == - HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (67) + +- Exchange (66) + +- HashAggregate (65) + +- Project (64) + +- Filter (63) + +- Scan parquet (62) -(47) Scan parquet +(50) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(48) NoopFilter +(51) NoopFilter Input [2]: [c_phone#X, c_acctbal#X] Arguments: [c_phone#X, c_acctbal#X] -(49) ProjectExecTransformer +(52) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(50) FlushableHashAggregateExecTransformer +(53) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(51) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(52) ColumnarExchange +(55) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(56) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(54) InputAdapter +(57) InputAdapter Input [2]: [sum#X, count#X] -(55) InputIteratorTransformer +(58) InputIteratorTransformer Input [2]: [sum#X, count#X] -(56) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(58) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(59) Scan parquet +(62) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(60) Filter +(63) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(61) Project +(64) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(62) HashAggregate +(65) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(63) Exchange +(66) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(67) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(65) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt index 0b11cfa1f763..8b1f048c7d6e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt @@ -1,56 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ NoopFilter (28) + +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (49) - : +- Project (48) - : +- ShuffledHashJoin Inner BuildLeft (47) - : :- Exchange (43) - : : +- Project (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Exchange (53) - +- Project (52) - +- Filter (51) - +- Scan parquet (50) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Project (59) + +- ShuffledHashJoin Inner BuildRight (58) + :- Exchange (53) + : +- Project (52) + : +- ShuffledHashJoin Inner BuildLeft (51) + : :- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Exchange (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -72,244 +76,260 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(23) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(42) Project +(46) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(43) Exchange +(47) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(48) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(45) Filter +(49) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(46) Exchange +(50) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(51) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(48) Project +(52) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Exchange +(53) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(54) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(51) Filter +(55) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(52) Project +(56) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(53) Exchange +(57) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(58) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(55) Project +(59) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(56) HashAggregate +(60) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(57) HashAggregate +(61) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt index 2eb1f1044104..054409875b25 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt @@ -1,47 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftSemi BuildRight (43) - :- Exchange (38) - : +- Project (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftSemi BuildRight (46) + :- Exchange (41) + : +- Project (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -63,200 +66,212 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [l_orderkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [l_orderkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(35) Scan parquet +(38) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(36) Filter +(39) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(37) Project +(40) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(38) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(41) Project +(44) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(42) Exchange +(45) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(44) Project +(47) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(45) HashAggregate +(48) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(46) Exchange +(49) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(50) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(48) Exchange +(51) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(52) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt index 85b46c76a499..db0ff094f920 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt @@ -1,115 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (134) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93), Statistics(X) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (133) - +- Exchange (132) - +- HashAggregate (131) - +- Exchange (130) - +- HashAggregate (129) - +- Project (128) - +- ShuffledHashJoin Inner BuildRight (127) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Project (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (126) - +- Project (125) - +- Filter (124) - +- Scan parquet (123) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Project (111) + : : : : +- Filter (110) + : : : : +- Scan parquet (109) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (137) + +- Project (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -131,552 +142,596 @@ Input [2]: [c_custkey#X, c_nationkey#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X + +(66) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: X + +(74) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [1]: [r_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [1]: [r_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(99) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [2]: [n_name#X, revenue#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) Exchange +(108) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(109) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(99) Filter +(110) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(100) Project +(111) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(101) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(103) Project +(114) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(104) Exchange +(115) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(107) Exchange +(118) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(109) Project +(120) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(110) Exchange +(121) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(113) Exchange +(124) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(115) Project +(126) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(116) Exchange +(127) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(119) Exchange +(130) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(121) Project +(132) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(122) Exchange +(133) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(125) Project +(136) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(126) Exchange +(137) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(128) Project +(139) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(129) HashAggregate +(140) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(130) Exchange +(141) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) HashAggregate +(142) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(132) Exchange +(143) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(144) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(134) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt index e03830992c2e..30e1b3fe013b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt index a2e4435d7188..5bccbe75ed31 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt @@ -1,110 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (138) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ SortExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ShuffleQueryStage (70), Statistics(X) - +- ReusedExchange (69) + VeloxColumnarToRowExec (100) + +- ^ SortExecTransformer (98) + +- ^ InputIteratorTransformer (97) + +- ShuffleQueryStage (95), Statistics(X) + +- ColumnarExchange (94) + +- VeloxAppendBatches (93) + +- ^ RegularHashAggregateExecTransformer (91) + +- ^ InputIteratorTransformer (90) + +- ShuffleQueryStage (88), Statistics(X) + +- ColumnarExchange (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ReusedExchange (78) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (111) - : : +- Project (110) - : : +- ShuffledHashJoin Inner BuildRight (109) - : : :- Exchange (105) - : : : +- Project (104) - : : : +- ShuffledHashJoin Inner BuildRight (103) - : : : :- Exchange (99) - : : : : +- Project (98) - : : : : +- ShuffledHashJoin Inner BuildLeft (97) - : : : : :- Exchange (93) - : : : : : +- Filter (92) - : : : : : +- Scan parquet (91) - : : : : +- Exchange (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Exchange (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (114) - : +- Filter (113) - : +- Scan parquet (112) - +- Exchange (120) - +- Filter (119) - +- Scan parquet (118) + Sort (137) + +- Exchange (136) + +- HashAggregate (135) + +- Exchange (134) + +- HashAggregate (133) + +- Project (132) + +- ShuffledHashJoin Inner BuildRight (131) + :- Exchange (127) + : +- Project (126) + : +- ShuffledHashJoin Inner BuildRight (125) + : :- Exchange (121) + : : +- Project (120) + : : +- ShuffledHashJoin Inner BuildRight (119) + : : :- Exchange (115) + : : : +- Project (114) + : : : +- ShuffledHashJoin Inner BuildRight (113) + : : : :- Exchange (109) + : : : : +- Project (108) + : : : : +- ShuffledHashJoin Inner BuildLeft (107) + : : : : :- Exchange (103) + : : : : : +- Filter (102) + : : : : : +- Scan parquet (101) + : : : : +- Exchange (106) + : : : : +- Filter (105) + : : : : +- Scan parquet (104) + : : : +- Exchange (112) + : : : +- Filter (111) + : : : +- Scan parquet (110) + : : +- Exchange (118) + : : +- Filter (117) + : : +- Scan parquet (116) + : +- Exchange (124) + : +- Filter (123) + : +- Scan parquet (122) + +- Exchange (130) + +- Filter (129) + +- Scan parquet (128) (1) Scan parquet @@ -126,524 +136,564 @@ Input [2]: [s_suppkey#X, s_nationkey#X] Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(23) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_orderkey#X, o_custkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: X + +(40) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(69) ReusedExchange [Reuses operator id: 58] +(78) ReusedExchange [Reuses operator id: 66] Output [2]: [n_nationkey#X, n_name#X] -(70) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(71) InputAdapter +(80) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(72) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(73) ShuffledHashJoinExecTransformer +(82) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(74) ProjectExecTransformer +(83) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(75) FlushableHashAggregateExecTransformer +(84) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) ProjectExecTransformer +(85) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) WholeStageCodegenTransformer (X) +(86) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(78) ColumnarExchange +(87) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(88) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(80) InputAdapter +(89) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(81) InputIteratorTransformer +(90) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(82) RegularHashAggregateExecTransformer +(91) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(83) WholeStageCodegenTransformer (X) +(92) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(84) ColumnarExchange +(93) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(94) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(95) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(86) InputAdapter +(96) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(87) InputIteratorTransformer +(97) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(88) SortExecTransformer +(98) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(89) WholeStageCodegenTransformer (X) +(99) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(90) VeloxColumnarToRowExec +(100) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(91) Scan parquet +(101) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(102) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(103) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(104) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(105) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Exchange +(106) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(98) Project +(108) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(102) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(104) Project +(114) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(105) Exchange +(115) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(116) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(117) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(118) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(110) Project +(120) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(111) Exchange +(121) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(122) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(113) Filter +(123) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(114) Exchange +(124) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(116) Project +(126) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(117) Exchange +(127) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(128) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(119) Filter +(129) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(120) Exchange +(130) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(122) Project +(132) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(123) HashAggregate +(133) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(124) Exchange +(134) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(135) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(126) Exchange +(136) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(137) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(128) AdaptiveSparkPlan +(138) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt index 47be7c7230dd..553bb62a242e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt @@ -1,150 +1,165 @@ == Physical Plan == -AdaptiveSparkPlan (177) +AdaptiveSparkPlan (192) +- == Final Plan == - VeloxColumnarToRowExec (125) - +- ^ SortExecTransformer (123) - +- ^ InputIteratorTransformer (122) - +- ShuffleQueryStage (120), Statistics(X) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ShuffleQueryStage (113), Statistics(X) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ShuffleQueryStage (96), Statistics(X) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ShuffleQueryStage (81), Statistics(X) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ShuffleQueryStage (66), Statistics(X) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ShuffleQueryStage (51), Statistics(X) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ShuffleQueryStage (36), Statistics(X) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ShuffleQueryStage (29), Statistics(X) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ NoopFilter (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ShuffleQueryStage (44), Statistics(X) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ NoopFilter (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ShuffleQueryStage (59), Statistics(X) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ NoopFilter (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ShuffleQueryStage (74), Statistics(X) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ NoopFilter (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ NoopFilter (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ShuffleQueryStage (104), Statistics(X) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ NoopFilter (100) - +- ^ Scan parquet (99) + VeloxColumnarToRowExec (140) + +- ^ SortExecTransformer (138) + +- ^ InputIteratorTransformer (137) + +- ShuffleQueryStage (135), Statistics(X) + +- ColumnarExchange (134) + +- VeloxAppendBatches (133) + +- ^ ProjectExecTransformer (131) + +- ^ RegularHashAggregateExecTransformer (130) + +- ^ InputIteratorTransformer (129) + +- ShuffleQueryStage (127), Statistics(X) + +- ColumnarExchange (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109), Statistics(X) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92), Statistics(X) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75), Statistics(X) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58), Statistics(X) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41), Statistics(X) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ NoopFilter (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33), Statistics(X) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ NoopFilter (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50), Statistics(X) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ NoopFilter (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67), Statistics(X) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ NoopFilter (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ NoopFilter (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ NoopFilter (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118), Statistics(X) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ NoopFilter (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (176) - +- Exchange (175) - +- HashAggregate (174) - +- Exchange (173) - +- HashAggregate (172) - +- Project (171) - +- ShuffledHashJoin Inner BuildRight (170) - :- Exchange (165) - : +- Project (164) - : +- ShuffledHashJoin Inner BuildRight (163) - : :- Exchange (159) - : : +- Project (158) - : : +- ShuffledHashJoin Inner BuildRight (157) - : : :- Exchange (153) - : : : +- Project (152) - : : : +- ShuffledHashJoin Inner BuildRight (151) - : : : :- Exchange (147) - : : : : +- Project (146) - : : : : +- ShuffledHashJoin Inner BuildRight (145) - : : : : :- Exchange (141) - : : : : : +- Project (140) - : : : : : +- ShuffledHashJoin Inner BuildRight (139) - : : : : : :- Exchange (135) - : : : : : : +- Project (134) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) - : : : : : : :- Exchange (129) - : : : : : : : +- Project (128) - : : : : : : : +- Filter (127) - : : : : : : : +- Scan parquet (126) - : : : : : : +- Exchange (132) - : : : : : : +- Filter (131) - : : : : : : +- Scan parquet (130) - : : : : : +- Exchange (138) - : : : : : +- Filter (137) - : : : : : +- Scan parquet (136) - : : : : +- Exchange (144) - : : : : +- Filter (143) - : : : : +- Scan parquet (142) - : : : +- Exchange (150) - : : : +- Filter (149) - : : : +- Scan parquet (148) - : : +- Exchange (156) - : : +- Filter (155) - : : +- Scan parquet (154) - : +- Exchange (162) - : +- Filter (161) - : +- Scan parquet (160) - +- Exchange (169) - +- Project (168) - +- Filter (167) - +- Scan parquet (166) + Sort (191) + +- Exchange (190) + +- HashAggregate (189) + +- Exchange (188) + +- HashAggregate (187) + +- Project (186) + +- ShuffledHashJoin Inner BuildRight (185) + :- Exchange (180) + : +- Project (179) + : +- ShuffledHashJoin Inner BuildRight (178) + : :- Exchange (174) + : : +- Project (173) + : : +- ShuffledHashJoin Inner BuildRight (172) + : : :- Exchange (168) + : : : +- Project (167) + : : : +- ShuffledHashJoin Inner BuildRight (166) + : : : :- Exchange (162) + : : : : +- Project (161) + : : : : +- ShuffledHashJoin Inner BuildRight (160) + : : : : :- Exchange (156) + : : : : : +- Project (155) + : : : : : +- ShuffledHashJoin Inner BuildRight (154) + : : : : : :- Exchange (150) + : : : : : : +- Project (149) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (148) + : : : : : : :- Exchange (144) + : : : : : : : +- Project (143) + : : : : : : : +- Filter (142) + : : : : : : : +- Scan parquet (141) + : : : : : : +- Exchange (147) + : : : : : : +- Filter (146) + : : : : : : +- Scan parquet (145) + : : : : : +- Exchange (153) + : : : : : +- Filter (152) + : : : : : +- Scan parquet (151) + : : : : +- Exchange (159) + : : : : +- Filter (158) + : : : : +- Scan parquet (157) + : : : +- Exchange (165) + : : : +- Filter (164) + : : : +- Scan parquet (163) + : : +- Exchange (171) + : : +- Filter (170) + : : +- Scan parquet (169) + : +- Exchange (177) + : +- Filter (176) + : +- Scan parquet (175) + +- Exchange (184) + +- Project (183) + +- Filter (182) + +- Scan parquet (181) (1) Scan parquet @@ -166,732 +181,792 @@ Input [2]: [p_partkey#X, p_type#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: [n_nationkey#X, n_regionkey#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(79) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(80) ColumnarExchange +(90) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: X + +(91) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(92) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(82) InputAdapter +(93) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(83) InputIteratorTransformer +(94) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(84) Scan parquet +(95) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) NoopFilter +(96) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(86) ProjectExecTransformer +(97) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(92) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(93) ProjectExecTransformer +(105) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(94) WholeStageCodegenTransformer (X) +(106) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(95) ColumnarExchange +(107) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: X + +(108) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(109) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(97) InputAdapter +(110) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(98) InputIteratorTransformer +(111) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(99) Scan parquet +(112) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(100) NoopFilter +(113) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(101) ProjectExecTransformer +(114) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(102) WholeStageCodegenTransformer (X) +(115) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(103) ColumnarExchange +(116) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(117) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(104) ShuffleQueryStage +(118) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(105) InputAdapter +(119) InputAdapter Input [1]: [r_regionkey#X] -(106) InputIteratorTransformer +(120) InputIteratorTransformer Input [1]: [r_regionkey#X] -(107) ShuffledHashJoinExecTransformer +(121) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(108) ProjectExecTransformer +(122) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(109) FlushableHashAggregateExecTransformer +(123) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(110) ProjectExecTransformer +(124) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(111) WholeStageCodegenTransformer (X) +(125) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(112) ColumnarExchange +(126) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(113) ShuffleQueryStage +(127) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(114) InputAdapter +(128) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(115) InputIteratorTransformer +(129) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(116) RegularHashAggregateExecTransformer +(130) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(117) ProjectExecTransformer +(131) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(118) WholeStageCodegenTransformer (X) +(132) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(119) ColumnarExchange +(133) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(134) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(120) ShuffleQueryStage +(135) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(121) InputAdapter +(136) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(122) InputIteratorTransformer +(137) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(123) SortExecTransformer +(138) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(124) WholeStageCodegenTransformer (X) +(139) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(125) VeloxColumnarToRowExec +(140) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(126) Scan parquet +(141) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(127) Filter +(142) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(128) Project +(143) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(129) Exchange +(144) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) Scan parquet +(145) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(131) Filter +(146) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(132) Exchange +(147) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) ShuffledHashJoin +(148) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(134) Project +(149) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(135) Exchange +(150) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Scan parquet +(151) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(137) Filter +(152) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(138) Exchange +(153) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(154) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(140) Project +(155) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(141) Exchange +(156) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) Scan parquet +(157) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(143) Filter +(158) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(144) Exchange +(159) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) ShuffledHashJoin +(160) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(146) Project +(161) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(147) Exchange +(162) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Scan parquet +(163) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(149) Filter +(164) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(150) Exchange +(165) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) ShuffledHashJoin +(166) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(152) Project +(167) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(153) Exchange +(168) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) Scan parquet +(169) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(155) Filter +(170) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(156) Exchange +(171) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) ShuffledHashJoin +(172) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(158) Project +(173) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(159) Exchange +(174) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Scan parquet +(175) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(161) Filter +(176) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(162) Exchange +(177) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) ShuffledHashJoin +(178) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(164) Project +(179) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(165) Exchange +(180) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) Scan parquet +(181) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(167) Filter +(182) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(168) Project +(183) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(169) Exchange +(184) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) ShuffledHashJoin +(185) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(171) Project +(186) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(172) HashAggregate +(187) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(173) Exchange +(188) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(174) HashAggregate +(189) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(175) Exchange +(190) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(191) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(177) AdaptiveSparkPlan +(192) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt index 93001ea6c78d..f894a33ab6e7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt @@ -1,114 +1,125 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (144) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93), Statistics(X) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- HashAggregate (130) - +- Exchange (129) - +- HashAggregate (128) - +- Project (127) - +- ShuffledHashJoin Inner BuildRight (126) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (98) - : : : : : +- Project (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (125) - +- Filter (124) - +- Scan parquet (123) + Sort (143) + +- Exchange (142) + +- HashAggregate (141) + +- Exchange (140) + +- HashAggregate (139) + +- Project (138) + +- ShuffledHashJoin Inner BuildRight (137) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (109) + : : : : : +- Project (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -130,548 +141,592 @@ Input [2]: [p_partkey#X, p_name#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: X + +(57) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_orderdate#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: X + +(74) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(99) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(97) Project +(108) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(109) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(110) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(100) Filter +(111) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(101) Exchange +(112) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(103) Project +(114) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(115) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(118) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(109) Project +(120) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(121) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(113) Exchange +(124) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(115) Project +(126) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(116) Exchange +(127) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(119) Exchange +(130) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(121) Project +(132) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(122) Exchange +(133) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(125) Exchange +(136) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(137) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(127) Project +(138) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(128) HashAggregate +(139) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(129) Exchange +(140) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) HashAggregate +(141) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(131) Exchange +(142) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(143) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(133) AdaptiveSparkPlan +(144) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt index c254ec8c82ca..589019c4cdbf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt index 43930a2eb5db..e26d0ed210e2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt @@ -1,78 +1,84 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- TakeOrderedAndProjectExecTransformer (59) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ShuffleQueryStage (36), Statistics(X) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ShuffleQueryStage (21), Statistics(X) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ShuffleQueryStage (29), Statistics(X) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ NoopFilter (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (66) + +- TakeOrderedAndProjectExecTransformer (65) + +- ^ ProjectExecTransformer (63) + +- ^ RegularHashAggregateExecTransformer (62) + +- ^ InputIteratorTransformer (61) + +- ShuffleQueryStage (59), Statistics(X) + +- ColumnarExchange (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24), Statistics(X) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ NoopFilter (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33), Statistics(X) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ NoopFilter (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ NoopFilter (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- ShuffledHashJoin Inner BuildRight (81) - :- Exchange (77) - : +- Project (76) - : +- ShuffledHashJoin Inner BuildRight (75) - : :- Exchange (70) - : : +- Project (69) - : : +- ShuffledHashJoin Inner BuildRight (68) - : : :- Exchange (63) - : : : +- Filter (62) - : : : +- Scan parquet (61) - : : +- Exchange (67) - : : +- Project (66) - : : +- Filter (65) - : : +- Scan parquet (64) - : +- Exchange (74) - : +- Project (73) - : +- Filter (72) - : +- Scan parquet (71) - +- Exchange (80) - +- Filter (79) - +- Scan parquet (78) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- ShuffledHashJoin Inner BuildRight (87) + :- Exchange (83) + : +- Project (82) + : +- ShuffledHashJoin Inner BuildRight (81) + : :- Exchange (76) + : : +- Project (75) + : : +- ShuffledHashJoin Inner BuildRight (74) + : : :- Exchange (69) + : : : +- Filter (68) + : : : +- Scan parquet (67) + : : +- Exchange (73) + : : +- Project (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (80) + : +- Project (79) + : +- Filter (78) + : +- Scan parquet (77) + +- Exchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -94,364 +100,388 @@ Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acct Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: X + +(6) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(55) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(56) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(58) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(59) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(60) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(61) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(62) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(57) ProjectExecTransformer +(63) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(58) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(59) TakeOrderedAndProjectExecTransformer +(65) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(60) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(61) Scan parquet +(67) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) Filter +(68) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(63) Exchange +(69) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(65) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(66) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(67) Exchange +(73) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) ShuffledHashJoin +(74) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(69) Project +(75) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(70) Exchange +(76) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(77) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(72) Filter +(78) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(73) Project +(79) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(74) Exchange +(80) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(81) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(76) Project +(82) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(77) Exchange +(83) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(80) Exchange +(86) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(87) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(82) Project +(88) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(89) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(84) Exchange +(90) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(91) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(86) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt index e8e0cb750c84..a360e2ccdf4d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt @@ -1,65 +1,70 @@ == Physical Plan == -AdaptiveSparkPlan (72) +AdaptiveSparkPlan (77) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ SortExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ShuffleQueryStage (38), Statistics(X) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (55) + +- ^ SortExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ FilterExecTransformer (46) + +- ^ RegularHashAggregateExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42), Statistics(X) + +- ColumnarExchange (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ NoopFilter (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (71) - +- Exchange (70) - +- Filter (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- ShuffledHashJoin Inner BuildRight (64) - :- Exchange (59) - : +- Project (58) - : +- ShuffledHashJoin Inner BuildRight (57) - : :- Exchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- Exchange (56) - : +- Filter (55) - : +- Scan parquet (54) - +- Exchange (63) - +- Project (62) - +- Filter (61) - +- Scan parquet (60) + Sort (76) + +- Exchange (75) + +- Filter (74) + +- HashAggregate (73) + +- Exchange (72) + +- HashAggregate (71) + +- Project (70) + +- ShuffledHashJoin Inner BuildRight (69) + :- Exchange (64) + : +- Project (63) + : +- ShuffledHashJoin Inner BuildRight (62) + : :- Exchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- Exchange (61) + : +- Filter (60) + : +- Scan parquet (59) + +- Exchange (68) + +- Project (67) + +- Filter (66) + +- Scan parquet (65) (1) Scan parquet @@ -81,557 +86,587 @@ Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [1]: [n_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [n_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(34) FlushableHashAggregateExecTransformer +(38) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(41) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(43) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(45) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(42) FilterExecTransformer +(46) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(43) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(44) ColumnarExchange +(48) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(49) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(46) InputAdapter +(51) InputAdapter Input [2]: [ps_partkey#X, value#X] -(47) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(48) SortExecTransformer +(53) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(49) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(50) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(51) Scan parquet +(56) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(52) Filter +(57) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(53) Exchange +(58) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(59) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(60) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) Exchange +(61) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(62) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(58) Project +(63) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(59) Exchange +(64) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(65) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(61) Filter +(66) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(62) Project +(67) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(63) Exchange +(68) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(69) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(65) Project +(70) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(66) HashAggregate +(71) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(67) Exchange +(72) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(73) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(69) Filter +(74) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(70) Exchange +(75) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(76) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(72) AdaptiveSparkPlan +(77) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (120) +Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (127) +- == Final Plan == - VeloxColumnarToRowExec (102) - +- ^ ProjectExecTransformer (100) - +- ^ RegularHashAggregateExecTransformer (99) - +- ^ RegularHashAggregateExecTransformer (98) - +- ^ ProjectExecTransformer (97) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) - :- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) - : :- ^ InputIteratorTransformer (80) - : : +- ShuffleQueryStage (78), Statistics(X) - : : +- ColumnarExchange (77) - : : +- ^ ProjectExecTransformer (75) - : : +- ^ NoopFilter (74) - : : +- ^ Scan parquet (73) - : +- ^ InputIteratorTransformer (84) - : +- ShuffleQueryStage (82), Statistics(X) - : +- ReusedExchange (81) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ReusedExchange (92) + VeloxColumnarToRowExec (109) + +- ^ ProjectExecTransformer (107) + +- ^ RegularHashAggregateExecTransformer (106) + +- ^ RegularHashAggregateExecTransformer (105) + +- ^ ProjectExecTransformer (104) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (103) + :- ^ InputIteratorTransformer (98) + : +- ShuffleQueryStage (96), Statistics(X) + : +- ColumnarExchange (95) + : +- VeloxAppendBatches (94) + : +- ^ ProjectExecTransformer (92) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (91) + : :- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ NoopFilter (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (90) + : +- ShuffleQueryStage (88), Statistics(X) + : +- ReusedExchange (87) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ReusedExchange (99) +- == Initial Plan == - HashAggregate (119) - +- HashAggregate (118) - +- Project (117) - +- ShuffledHashJoin Inner BuildRight (116) - :- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildRight (109) - : :- Exchange (105) - : : +- Filter (104) - : : +- Scan parquet (103) - : +- Exchange (108) - : +- Filter (107) - : +- Scan parquet (106) - +- Exchange (115) - +- Project (114) - +- Filter (113) - +- Scan parquet (112) - - -(73) Scan parquet + HashAggregate (126) + +- HashAggregate (125) + +- Project (124) + +- ShuffledHashJoin Inner BuildRight (123) + :- Exchange (118) + : +- Project (117) + : +- ShuffledHashJoin Inner BuildRight (116) + : :- Exchange (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- Exchange (115) + : +- Filter (114) + : +- Scan parquet (113) + +- Exchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) + + +(78) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(74) NoopFilter +(79) NoopFilter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(75) ProjectExecTransformer +(80) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(76) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(77) ColumnarExchange +(82) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(83) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(78) ShuffleQueryStage +(84) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(79) InputAdapter +(85) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(80) InputIteratorTransformer +(86) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(81) ReusedExchange [Reuses operator id: 13] +(87) ReusedExchange [Reuses operator id: 15] Output [2]: [s_suppkey#X, s_nationkey#X] -(82) ShuffleQueryStage +(88) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(83) InputAdapter +(89) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(84) InputIteratorTransformer +(90) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(85) ShuffledHashJoinExecTransformer +(91) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(86) ProjectExecTransformer +(92) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(87) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(88) ColumnarExchange +(94) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(95) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(96) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(90) InputAdapter +(97) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(91) InputIteratorTransformer +(98) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(92) ReusedExchange [Reuses operator id: 28] +(99) ReusedExchange [Reuses operator id: 32] Output [1]: [n_nationkey#X] -(93) ShuffleQueryStage +(100) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(94) InputAdapter +(101) InputAdapter Input [1]: [n_nationkey#X] -(95) InputIteratorTransformer +(102) InputIteratorTransformer Input [1]: [n_nationkey#X] -(96) ShuffledHashJoinExecTransformer +(103) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(97) ProjectExecTransformer +(104) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(98) RegularHashAggregateExecTransformer +(105) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(99) RegularHashAggregateExecTransformer +(106) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(100) ProjectExecTransformer +(107) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(101) WholeStageCodegenTransformer (X) +(108) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(102) VeloxColumnarToRowExec +(109) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(103) Scan parquet +(110) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(104) Filter +(111) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(105) Exchange +(112) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(113) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(107) Filter +(114) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(108) Exchange +(115) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(116) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(110) Project +(117) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(111) Exchange +(118) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(119) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(113) Filter +(120) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(114) Project +(121) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(115) Exchange +(122) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(123) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(117) Project +(124) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(118) HashAggregate +(125) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(119) HashAggregate +(126) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(120) AdaptiveSparkPlan +(127) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt index ea49c8c6402c..2cd2fbbd94c0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt @@ -1,46 +1,49 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin Inner BuildLeft (42) - :- Exchange (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- ShuffledHashJoin Inner BuildLeft (45) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -62,198 +65,210 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(31) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) Scan parquet +(38) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(36) Filter +(39) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(37) Exchange +(40) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(41) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(40) Project +(43) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(41) Exchange +(44) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(45) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(43) Project +(46) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(44) HashAggregate +(47) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(45) Exchange +(48) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(49) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(47) Exchange +(50) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(51) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt index a7a55ece84ee..7e296f215aa1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt @@ -1,49 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftOuter BuildRight (43) - :- Exchange (38) - : +- Scan parquet (37) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftOuter BuildRight (46) + :- Exchange (41) + : +- Scan parquet (40) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -60,225 +63,237 @@ Input [1]: [c_custkey#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(4) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) NoopFilter +(10) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(44) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) Exchange +(45) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(44) Project +(47) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(48) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt index 412a680dae78..4080a469e711 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (23) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (25) + +- ^ ProjectExecTransformer (23) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (34) - +- HashAggregate (33) - +- Project (32) - +- ShuffledHashJoin Inner BuildRight (31) - :- Exchange (27) - : +- Project (26) - : +- Filter (25) - : +- Scan parquet (24) - +- Exchange (30) - +- Filter (29) - +- Scan parquet (28) + HashAggregate (36) + +- HashAggregate (35) + +- Project (34) + +- ShuffledHashJoin Inner BuildRight (33) + :- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -52,146 +54,154 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [2]: [p_partkey#X, p_type#X] Arguments: [p_partkey#X, p_type#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(21) ProjectExecTransformer +(23) ProjectExecTransformer Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(22) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(23) VeloxColumnarToRowExec +(25) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(24) Scan parquet +(26) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(26) Project +(28) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) Exchange +(29) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Scan parquet +(30) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(30) Exchange +(32) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) ShuffledHashJoin +(33) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(32) Project +(34) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(34) HashAggregate +(36) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] -(35) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt index c66afda3aee4..3fa54d1d647b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt @@ -1,43 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- AQEShuffleRead (26) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (29) + +- AQEShuffleRead (28) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (22) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- Project (39) - +- ShuffledHashJoin Inner BuildLeft (38) - :- Exchange (30) - : +- Filter (29) - : +- Scan parquet (28) - +- Filter (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (43) + +- Exchange (42) + +- Project (41) + +- ShuffledHashJoin Inner BuildLeft (40) + :- Exchange (32) + : +- Filter (31) + : +- Scan parquet (30) + +- Filter (39) + +- HashAggregate (38) + +- Exchange (37) + +- HashAggregate (36) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -59,330 +61,338 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(16) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(20) FilterExecTransformer +(21) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(21) ShuffledHashJoinExecTransformer +(22) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(26) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) AQEShuffleRead +(28) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(27) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) Scan parquet +(30) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(30) Exchange +(32) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) Scan parquet +(33) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(32) Filter +(34) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(33) Project +(35) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(34) HashAggregate +(36) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(35) Exchange +(37) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(38) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(37) Filter +(39) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(38) ShuffledHashJoin +(40) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(39) Project +(41) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(40) Exchange +(42) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(43) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (67) +Subquery:1 Hosting operator id = 21 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ RegularHashAggregateExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ ProjectExecTransformer (47) - +- ^ FlushableHashAggregateExecTransformer (46) - +- ^ ProjectExecTransformer (45) - +- ^ NoopFilter (44) - +- ^ Scan parquet (43) + VeloxColumnarToRowExec (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ ProjectExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ InputIteratorTransformer (54) + +- ShuffleQueryStage (52), Statistics(X) + +- ColumnarExchange (51) + +- ^ ProjectExecTransformer (49) + +- ^ FlushableHashAggregateExecTransformer (48) + +- ^ ProjectExecTransformer (47) + +- ^ NoopFilter (46) + +- ^ Scan parquet (45) +- == Initial Plan == - HashAggregate (66) - +- HashAggregate (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (68) + +- HashAggregate (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- Filter (62) + +- Scan parquet (61) -(43) Scan parquet +(45) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(44) NoopFilter +(46) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(45) ProjectExecTransformer +(47) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) FlushableHashAggregateExecTransformer +(48) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(48) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(49) ColumnarExchange +(51) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(51) InputAdapter +(53) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(53) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(54) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(55) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(56) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(57) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(58) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(59) Scan parquet +(61) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(60) Filter +(62) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(61) Project +(63) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(63) Exchange +(65) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(65) HashAggregate +(67) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(66) HashAggregate +(68) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(67) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt index 7bb61df21db4..70fe9d3d51e4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt @@ -1,59 +1,62 @@ == Physical Plan == -AdaptiveSparkPlan (64) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (42) - +- ^ SortExecTransformer (40) - +- ^ InputIteratorTransformer (39) - +- ShuffleQueryStage (37), Statistics(X) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (45) + +- ^ SortExecTransformer (43) + +- ^ InputIteratorTransformer (42) + +- ShuffleQueryStage (40), Statistics(X) + +- ColumnarExchange (39) + +- VeloxAppendBatches (38) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- ^ ProjectExecTransformer (30) + +- ^ FlushableHashAggregateExecTransformer (29) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (63) - +- Exchange (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- HashAggregate (58) - +- Exchange (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (50) - : +- BroadcastHashJoin LeftAnti BuildRight (49) - : :- Filter (44) - : : +- Scan parquet (43) - : +- BroadcastExchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (53) - +- Filter (52) - +- Scan parquet (51) + Sort (66) + +- Exchange (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- HashAggregate (61) + +- Exchange (60) + +- HashAggregate (59) + +- Project (58) + +- ShuffledHashJoin Inner BuildRight (57) + :- Exchange (53) + : +- BroadcastHashJoin LeftAnti BuildRight (52) + : :- Filter (47) + : : +- Scan parquet (46) + : +- BroadcastExchange (51) + : +- Project (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -75,273 +78,285 @@ Input [2]: [ps_partkey#X, ps_suppkey#X] Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) FlushableHashAggregateExecTransformer +(29) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(30) ColumnarExchange +(32) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(32) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) ColumnarExchange +(38) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(39) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(40) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(38) InputAdapter +(41) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) InputIteratorTransformer +(42) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(40) SortExecTransformer +(43) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(41) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(42) VeloxColumnarToRowExec +(45) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) Scan parquet +(46) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(45) Scan parquet +(48) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(46) Filter +(49) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(47) Project +(50) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(48) BroadcastExchange +(51) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(49) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: LeftAnti Join condition: None -(50) Exchange +(53) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(54) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(53) Exchange +(56) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(57) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(55) Project +(58) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(56) HashAggregate +(59) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) HashAggregate +(61) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(59) HashAggregate +(62) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(60) Exchange +(63) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(62) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) Sort +(66) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(64) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt index c51280b11b0f..b5360f940716 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt @@ -1,56 +1,58 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ ProjectExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) - :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ FilterExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ NoopFilter (20) - +- ^ Scan parquet (19) + VeloxColumnarToRowExec (39) + +- ^ ProjectExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ ProjectExecTransformer (34) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (33) + :- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ NoopFilter (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (56) - +- HashAggregate (55) - +- Project (54) - +- ShuffledHashJoin Inner BuildRight (53) - :- Project (46) - : +- ShuffledHashJoin Inner BuildRight (45) - : :- Exchange (40) - : : +- Filter (39) - : : +- Scan parquet (38) - : +- Exchange (44) - : +- Project (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Filter (48) - +- Scan parquet (47) + HashAggregate (58) + +- HashAggregate (57) + +- Project (56) + +- ShuffledHashJoin Inner BuildRight (55) + :- Project (48) + : +- ShuffledHashJoin Inner BuildRight (47) + : :- Exchange (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Exchange (46) + : +- Project (45) + : +- Filter (44) + : +- Scan parquet (43) + +- Filter (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Filter (50) + +- Scan parquet (49) (1) Scan parquet @@ -72,254 +74,262 @@ Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_container#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(19) Scan parquet +(21) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(20) NoopFilter +(22) NoopFilter Input [2]: [l_partkey#X, l_quantity#X] Arguments: [l_partkey#X, l_quantity#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(30) FilterExecTransformer +(32) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(31) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(32) ProjectExecTransformer +(34) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(33) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(35) ProjectExecTransformer +(37) ProjectExecTransformer Output [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(36) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(37) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(38) Scan parquet +(40) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(39) Filter +(41) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(40) Exchange +(42) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(43) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(43) Project +(45) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(44) Exchange +(46) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(46) Project +(48) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(47) Scan parquet +(49) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(48) Filter +(50) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(49) HashAggregate +(51) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(50) Exchange +(52) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) Filter +(54) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(53) ShuffledHashJoin +(55) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(54) Project +(56) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) HashAggregate +(57) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(56) HashAggregate +(58) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] -(57) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt index 0f3bc73eb825..557e4f5258cc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt @@ -1,89 +1,94 @@ == Physical Plan == -AdaptiveSparkPlan (97) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- TakeOrderedAndProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) - :- ^ InputIteratorTransformer (41) - : +- ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ShuffleQueryStage (22), Statistics(X) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) - :- ^ InputIteratorTransformer (49) - : +- ShuffleQueryStage (47), Statistics(X) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ NoopFilter (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ShuffleQueryStage (51), Statistics(X) - +- ReusedExchange (50) + VeloxColumnarToRowExec (69) + +- TakeOrderedAndProjectExecTransformer (68) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ RegularHashAggregateExecTransformer (65) + +- ^ ProjectExecTransformer (64) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (63) + :- ^ InputIteratorTransformer (45) + : +- ShuffleQueryStage (43), Statistics(X) + : +- ColumnarExchange (42) + : +- VeloxAppendBatches (41) + : +- ^ ProjectExecTransformer (39) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (38) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (37) + : +- ShuffleQueryStage (35), Statistics(X) + : +- ColumnarExchange (34) + : +- VeloxAppendBatches (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (30) + : :- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ NoopFilter (11) + : : +- ^ Scan parquet (10) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ RegularHashAggregateExecTransformer (27) + : +- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) + :- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- VeloxAppendBatches (50) + : +- ^ ProjectExecTransformer (48) + : +- ^ NoopFilter (47) + : +- ^ Scan parquet (46) + +- ^ ProjectExecTransformer (61) + +- ^ FilterExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ReusedExchange (55) +- == Initial Plan == - TakeOrderedAndProject (96) - +- HashAggregate (95) - +- HashAggregate (94) - +- Project (93) - +- ShuffledHashJoin Inner BuildRight (92) - :- Exchange (81) - : +- Project (80) - : +- ShuffledHashJoin Inner BuildLeft (79) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (78) - : +- ShuffledHashJoin LeftSemi BuildRight (77) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Project (76) - : +- Filter (75) - : +- HashAggregate (74) - : +- Exchange (73) - : +- HashAggregate (72) - : +- Scan parquet (71) - +- ShuffledHashJoin LeftSemi BuildRight (91) - :- Exchange (84) - : +- Filter (83) - : +- Scan parquet (82) - +- Project (90) - +- Filter (89) - +- HashAggregate (88) - +- Exchange (87) - +- HashAggregate (86) - +- Scan parquet (85) + TakeOrderedAndProject (101) + +- HashAggregate (100) + +- HashAggregate (99) + +- Project (98) + +- ShuffledHashJoin Inner BuildRight (97) + :- Exchange (86) + : +- Project (85) + : +- ShuffledHashJoin Inner BuildLeft (84) + : :- Exchange (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (83) + : +- ShuffledHashJoin LeftSemi BuildRight (82) + : :- Exchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- Project (81) + : +- Filter (80) + : +- HashAggregate (79) + : +- Exchange (78) + : +- HashAggregate (77) + : +- Scan parquet (76) + +- ShuffledHashJoin LeftSemi BuildRight (96) + :- Exchange (89) + : +- Filter (88) + : +- Scan parquet (87) + +- Project (95) + +- Filter (94) + +- HashAggregate (93) + +- Exchange (92) + +- HashAggregate (91) + +- Scan parquet (90) (1) Scan parquet @@ -105,428 +110,448 @@ Input [2]: [c_custkey#X, c_name#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(26) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(27) ProjectExecTransformer +(29) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(28) ShuffledHashJoinExecTransformer +(30) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(30) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(31) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(33) InputAdapter +(36) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(34) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) ShuffledHashJoinExecTransformer +(38) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(36) ProjectExecTransformer +(39) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(38) ColumnarExchange +(41) VeloxAppendBatches +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(42) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(43) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(40) InputAdapter +(44) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(41) InputIteratorTransformer +(45) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(42) Scan parquet +(46) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(43) NoopFilter +(47) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(46) ColumnarExchange +(50) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: X + +(51) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(52) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(48) InputAdapter +(53) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(49) InputIteratorTransformer +(54) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(50) ReusedExchange [Reuses operator id: 21] +(55) ReusedExchange [Reuses operator id: 23] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) ShuffleQueryStage +(56) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(52) InputAdapter +(57) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(53) InputIteratorTransformer +(58) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(54) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(55) FilterExecTransformer +(60) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(56) ProjectExecTransformer +(61) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(58) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(59) ProjectExecTransformer +(64) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(60) RegularHashAggregateExecTransformer +(65) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(61) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(62) WholeStageCodegenTransformer (X) +(67) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(63) TakeOrderedAndProjectExecTransformer +(68) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(64) VeloxColumnarToRowExec +(69) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(65) Scan parquet +(70) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(66) Filter +(71) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(67) Exchange +(72) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(73) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(69) Filter +(74) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(70) Exchange +(75) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(76) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(77) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(78) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(79) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(80) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(81) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(78) Exchange +(83) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) ShuffledHashJoin +(84) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(80) Project +(85) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(81) Exchange +(86) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(87) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(83) Filter +(88) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(84) Exchange +(89) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(90) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(86) HashAggregate +(91) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(87) Exchange +(92) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) HashAggregate +(93) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(89) Filter +(94) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(90) Project +(95) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(91) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(93) Project +(98) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(94) HashAggregate +(99) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(95) HashAggregate +(100) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(96) TakeOrderedAndProject +(101) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(97) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt index 1d1169055f32..91187ac8d5a7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (33) - +- HashAggregate (32) - +- Project (31) - +- ShuffledHashJoin Inner BuildRight (30) - :- Exchange (26) - : +- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- Exchange (29) - +- Filter (28) - +- Scan parquet (27) + HashAggregate (35) + +- HashAggregate (34) + +- Project (33) + +- ShuffledHashJoin Inner BuildRight (32) + :- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) (1) Scan parquet @@ -51,142 +53,150 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(24) VeloxColumnarToRowExec Input [1]: [revenue#X] -(23) Scan parquet +(25) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(24) Filter +(26) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(25) Project +(27) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(26) Exchange +(28) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Scan parquet +(29) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(28) Filter +(30) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(29) Exchange +(31) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) ShuffledHashJoin +(32) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(31) Project +(33) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(32) HashAggregate +(34) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(34) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt index 71c30b25fb2b..01ab88ee0b2b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt @@ -1,109 +1,119 @@ == Physical Plan == -AdaptiveSparkPlan (123) +AdaptiveSparkPlan (133) +- == Final Plan == - VeloxColumnarToRowExec (83) - +- AQEShuffleRead (82) - +- ShuffleQueryStage (81), Statistics(X) - +- ColumnarExchange (80) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ShuffleQueryStage (29), Statistics(X) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ShuffleQueryStage (14), Statistics(X) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ NoopFilter (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ShuffleQueryStage (22), Statistics(X) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ NoopFilter (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ NoopFilter (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41), Statistics(X) - : +- ReusedExchange (40) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (93) + +- AQEShuffleRead (92) + +- ShuffleQueryStage (91), Statistics(X) + +- ColumnarExchange (90) + +- VeloxAppendBatches (89) + +- ^ ProjectExecTransformer (87) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (86) + :- ^ InputIteratorTransformer (76) + : +- ShuffleQueryStage (74), Statistics(X) + : +- ColumnarExchange (73) + : +- VeloxAppendBatches (72) + : +- ^ ProjectExecTransformer (70) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (69) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- VeloxAppendBatches (64) + : +- ^ ProjectExecTransformer (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (61) + : :- ^ InputIteratorTransformer (35) + : : +- ShuffleQueryStage (33), Statistics(X) + : : +- ColumnarExchange (32) + : : +- VeloxAppendBatches (31) + : : +- ^ ProjectExecTransformer (29) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : : :- ^ InputIteratorTransformer (18) + : : : +- ShuffleQueryStage (16), Statistics(X) + : : : +- ColumnarExchange (15) + : : : +- VeloxAppendBatches (14) + : : : +- ^ ProjectExecTransformer (12) + : : : +- ^ NoopFilter (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (27) + : : +- ShuffleQueryStage (25), Statistics(X) + : : +- ColumnarExchange (24) + : : +- VeloxAppendBatches (23) + : : +- ^ ProjectExecTransformer (21) + : : +- ^ NoopFilter (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (60) + : +- ShuffleQueryStage (58), Statistics(X) + : +- ColumnarExchange (57) + : +- VeloxAppendBatches (56) + : +- ^ ProjectExecTransformer (54) + : +- ^ FilterExecTransformer (53) + : +- ^ ProjectExecTransformer (52) + : +- ^ RegularHashAggregateExecTransformer (51) + : +- ^ RegularHashAggregateExecTransformer (50) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (49) + : :- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ NoopFilter (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (48) + : +- ShuffleQueryStage (46), Statistics(X) + : +- ReusedExchange (45) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- VeloxAppendBatches (81) + +- ^ ProjectExecTransformer (79) + +- ^ NoopFilter (78) + +- ^ Scan parquet (77) +- == Initial Plan == - Sort (122) - +- Exchange (121) - +- Project (120) - +- ShuffledHashJoin Inner BuildRight (119) - :- Exchange (114) - : +- Project (113) - : +- ShuffledHashJoin LeftSemi BuildRight (112) - : :- Exchange (86) - : : +- Filter (85) - : : +- Scan parquet (84) - : +- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildLeft (109) - : :- Exchange (95) - : : +- ShuffledHashJoin LeftSemi BuildRight (94) - : : :- Exchange (89) - : : : +- Filter (88) - : : : +- Scan parquet (87) - : : +- Exchange (93) - : : +- Project (92) - : : +- Filter (91) - : : +- Scan parquet (90) - : +- Exchange (108) - : +- Filter (107) - : +- HashAggregate (106) - : +- HashAggregate (105) - : +- ShuffledHashJoin LeftSemi BuildRight (104) - : :- Exchange (99) - : : +- Project (98) - : : +- Filter (97) - : : +- Scan parquet (96) - : +- Exchange (103) - : +- Project (102) - : +- Filter (101) - : +- Scan parquet (100) - +- Exchange (118) - +- Project (117) - +- Filter (116) - +- Scan parquet (115) + Sort (132) + +- Exchange (131) + +- Project (130) + +- ShuffledHashJoin Inner BuildRight (129) + :- Exchange (124) + : +- Project (123) + : +- ShuffledHashJoin LeftSemi BuildRight (122) + : :- Exchange (96) + : : +- Filter (95) + : : +- Scan parquet (94) + : +- Exchange (121) + : +- Project (120) + : +- ShuffledHashJoin Inner BuildLeft (119) + : :- Exchange (105) + : : +- ShuffledHashJoin LeftSemi BuildRight (104) + : : :- Exchange (99) + : : : +- Filter (98) + : : : +- Scan parquet (97) + : : +- Exchange (103) + : : +- Project (102) + : : +- Filter (101) + : : +- Scan parquet (100) + : +- Exchange (118) + : +- Filter (117) + : +- HashAggregate (116) + : +- HashAggregate (115) + : +- ShuffledHashJoin LeftSemi BuildRight (114) + : :- Exchange (109) + : : +- Project (108) + : : +- Filter (107) + : : +- Scan parquet (106) + : +- Exchange (113) + : +- Project (112) + : +- Filter (111) + : +- Scan parquet (110) + +- Exchange (128) + +- Project (127) + +- Filter (126) + +- Scan parquet (125) (1) Scan parquet @@ -125,518 +135,558 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(18) NoopFilter +(20) NoopFilter Input [2]: [p_partkey#X, p_name#X] Arguments: [p_partkey#X, p_name#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [1]: [p_partkey#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [1]: [p_partkey#X] -(25) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(32) Scan parquet +(36) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(33) NoopFilter +(37) NoopFilter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(40) ReusedExchange [Reuses operator id: 21] +(45) ReusedExchange [Reuses operator id: 24] Output [1]: [p_partkey#X] -(41) ShuffleQueryStage +(46) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(42) InputAdapter +(47) InputAdapter Input [1]: [p_partkey#X] -(43) InputIteratorTransformer +(48) InputIteratorTransformer Input [1]: [p_partkey#X] -(44) ShuffledHashJoinExecTransformer +(49) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(45) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(46) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(47) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(48) FilterExecTransformer +(53) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(49) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(51) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(53) InputAdapter +(59) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(54) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(55) ShuffledHashJoinExecTransformer +(61) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(58) ColumnarExchange +(64) VeloxAppendBatches +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: X + +(65) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(66) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(60) InputAdapter +(67) InputAdapter Input [1]: [ps_suppkey#X] -(61) InputIteratorTransformer +(68) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(62) ShuffledHashJoinExecTransformer +(69) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(63) ProjectExecTransformer +(70) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(71) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(72) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(73) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(74) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(75) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(68) InputIteratorTransformer +(76) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(69) Scan parquet +(77) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(70) NoopFilter +(78) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(71) ProjectExecTransformer +(79) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(73) ColumnarExchange +(81) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(82) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(83) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(75) InputAdapter +(84) InputAdapter Input [1]: [n_nationkey#X] -(76) InputIteratorTransformer +(85) InputIteratorTransformer Input [1]: [n_nationkey#X] -(77) ShuffledHashJoinExecTransformer +(86) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(87) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(79) WholeStageCodegenTransformer (X) +(88) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(80) ColumnarExchange +(89) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(90) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(91) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(82) AQEShuffleRead +(92) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(83) VeloxColumnarToRowExec +(93) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(84) Scan parquet +(94) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(85) Filter +(95) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(86) Exchange +(96) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Scan parquet +(97) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(88) Filter +(98) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(89) Exchange +(99) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(100) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(91) Filter +(101) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(92) Project +(102) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(93) Exchange +(103) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) ShuffledHashJoin +(104) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(95) Exchange +(105) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) Scan parquet +(106) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(97) Filter +(107) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(98) Project +(108) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(102) Project +(112) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(103) Exchange +(113) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(105) HashAggregate +(115) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(106) HashAggregate +(116) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(107) Filter +(117) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(108) Exchange +(118) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(110) Project +(120) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(111) Exchange +(121) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(113) Project +(123) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(114) Exchange +(124) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) Scan parquet +(125) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(116) Filter +(126) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(117) Project +(127) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(118) Exchange +(128) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(129) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(120) Project +(130) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(121) Exchange +(131) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Sort +(132) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(123) AdaptiveSparkPlan +(133) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt index d9d79a967f03..f57dfa3e6f73 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt @@ -1,103 +1,112 @@ == Physical Plan == -AdaptiveSparkPlan (118) +AdaptiveSparkPlan (127) +- == Final Plan == - VeloxColumnarToRowExec (81) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ShuffleQueryStage (21), Statistics(X) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ShuffleQueryStage (30), Statistics(X) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ NoopFilter (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ NoopFilter (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ShuffleQueryStage (67), Statistics(X) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ NoopFilter (63) - +- ^ Scan parquet (62) + VeloxColumnarToRowExec (90) + +- ^ RegularHashAggregateExecTransformer (88) + +- ^ InputIteratorTransformer (87) + +- ShuffleQueryStage (85), Statistics(X) + +- ColumnarExchange (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24), Statistics(X) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34), Statistics(X) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ NoopFilter (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ NoopFilter (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ NoopFilter (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (117) - +- HashAggregate (116) - +- Exchange (115) - +- HashAggregate (114) - +- Project (113) - +- ShuffledHashJoin Inner BuildRight (112) - :- Exchange (107) - : +- Project (106) - : +- ShuffledHashJoin Inner BuildRight (105) - : :- Exchange (100) - : : +- Project (99) - : : +- ShuffledHashJoin Inner BuildLeft (98) - : : :- Exchange (84) - : : : +- Filter (83) - : : : +- Scan parquet (82) - : : +- Exchange (97) - : : +- ShuffledHashJoin LeftAnti BuildRight (96) - : : :- ShuffledHashJoin LeftSemi BuildRight (91) - : : : :- Exchange (88) - : : : : +- Project (87) - : : : : +- Filter (86) - : : : : +- Scan parquet (85) - : : : +- Exchange (90) - : : : +- Scan parquet (89) - : : +- Exchange (95) - : : +- Project (94) - : : +- Filter (93) - : : +- Scan parquet (92) - : +- Exchange (104) - : +- Project (103) - : +- Filter (102) - : +- Scan parquet (101) - +- Exchange (111) - +- Project (110) - +- Filter (109) - +- Scan parquet (108) + TakeOrderedAndProject (126) + +- HashAggregate (125) + +- Exchange (124) + +- HashAggregate (123) + +- Project (122) + +- ShuffledHashJoin Inner BuildRight (121) + :- Exchange (116) + : +- Project (115) + : +- ShuffledHashJoin Inner BuildRight (114) + : :- Exchange (109) + : : +- Project (108) + : : +- ShuffledHashJoin Inner BuildLeft (107) + : : :- Exchange (93) + : : : +- Filter (92) + : : : +- Scan parquet (91) + : : +- Exchange (106) + : : +- ShuffledHashJoin LeftAnti BuildRight (105) + : : :- ShuffledHashJoin LeftSemi BuildRight (100) + : : : :- Exchange (97) + : : : : +- Project (96) + : : : : +- Filter (95) + : : : : +- Scan parquet (94) + : : : +- Exchange (99) + : : : +- Scan parquet (98) + : : +- Exchange (104) + : : +- Project (103) + : : +- Filter (102) + : : +- Scan parquet (101) + : +- Exchange (113) + : +- Project (112) + : +- Filter (111) + : +- Scan parquet (110) + +- Exchange (120) + +- Project (119) + +- Filter (118) + +- Scan parquet (117) (1) Scan parquet @@ -119,500 +128,536 @@ Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(24) ShuffledHashJoinExecTransformer +(27) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(25) Scan parquet +(28) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(26) NoopFilter +(29) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(29) ColumnarExchange +(32) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(33) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(33) ShuffledHashJoinExecTransformer +(37) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(41) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(40) ShuffledHashJoinExecTransformer +(45) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(47) Scan parquet +(53) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(48) NoopFilter +(54) NoopFilter Input [2]: [o_orderkey#X, o_orderstatus#X] Arguments: [o_orderkey#X, o_orderstatus#X] -(49) ProjectExecTransformer +(55) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(50) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(51) ColumnarExchange +(57) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: X + +(58) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(53) InputAdapter +(60) InputAdapter Input [1]: [o_orderkey#X] -(54) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [o_orderkey#X] -(55) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(62) Scan parquet +(70) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(63) NoopFilter +(71) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(64) ProjectExecTransformer +(72) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(65) WholeStageCodegenTransformer (X) +(73) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(66) ColumnarExchange +(74) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(75) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(76) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(68) InputAdapter +(77) InputAdapter Input [1]: [n_nationkey#X] -(69) InputIteratorTransformer +(78) InputIteratorTransformer Input [1]: [n_nationkey#X] -(70) ShuffledHashJoinExecTransformer +(79) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(72) FlushableHashAggregateExecTransformer +(81) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(73) ProjectExecTransformer +(82) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(74) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(75) ColumnarExchange +(84) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(85) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(77) InputAdapter +(86) InputAdapter Input [2]: [s_name#X, count#X] -(78) InputIteratorTransformer +(87) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(79) RegularHashAggregateExecTransformer +(88) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(80) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(81) VeloxColumnarToRowExec +(90) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(82) Scan parquet +(91) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(83) Filter +(92) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(84) Exchange +(93) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(94) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(86) Filter +(95) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(87) Project +(96) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(88) Exchange +(97) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) Scan parquet +(98) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(90) Exchange +(99) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) ShuffledHashJoin +(100) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(92) Scan parquet +(101) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(93) Filter +(102) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(94) Project +(103) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(95) Exchange +(104) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(105) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(97) Exchange +(106) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(99) Project +(108) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(100) Exchange +(109) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Scan parquet +(110) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(102) Filter +(111) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(103) Project +(112) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(104) Exchange +(113) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(106) Project +(115) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(107) Exchange +(116) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Scan parquet +(117) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(109) Filter +(118) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(110) Project +(119) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(111) Exchange +(120) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(121) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(113) Project +(122) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(114) HashAggregate +(123) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(115) Exchange +(124) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) HashAggregate +(125) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(117) TakeOrderedAndProject +(126) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(118) AdaptiveSparkPlan +(127) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt index feeda6c9fded..1e1b477c5c30 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt @@ -1,43 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (46) +AdaptiveSparkPlan (49) +- == Final Plan == - VeloxColumnarToRowExec (33) - +- ^ SortExecTransformer (31) - +- ^ InputIteratorTransformer (30) - +- ShuffleQueryStage (28), Statistics(X) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (45) - +- Exchange (44) - +- HashAggregate (43) - +- Exchange (42) - +- HashAggregate (41) - +- Project (40) - +- ShuffledHashJoin LeftAnti BuildRight (39) - :- Exchange (36) - : +- Filter (35) - : +- Scan parquet (34) - +- Exchange (38) - +- Scan parquet (37) + Sort (48) + +- Exchange (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- ShuffledHashJoin LeftAnti BuildRight (42) + :- Exchange (39) + : +- Filter (38) + : +- Scan parquet (37) + +- Exchange (41) + +- Scan parquet (40) (1) Scan parquet @@ -59,298 +62,310 @@ Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) Scan parquet +(10) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [1]: [o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [1]: [o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(26) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) ColumnarExchange +(29) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(30) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(28) ShuffleQueryStage +(31) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(29) InputAdapter +(32) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) InputIteratorTransformer +(33) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(31) SortExecTransformer +(34) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(32) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(33) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) Scan parquet +(37) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(35) Filter +(38) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(36) Exchange +(39) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(40) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(40) Project +(43) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(41) HashAggregate +(44) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(42) Exchange +(45) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) HashAggregate +(46) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(44) Exchange +(47) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(48) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(46) AdaptiveSparkPlan +(49) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (65) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ NoopFilter (48) - +- ^ Scan parquet (47) + VeloxColumnarToRowExec (61) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- ^ FlushableHashAggregateExecTransformer (53) + +- ^ ProjectExecTransformer (52) + +- ^ NoopFilter (51) + +- ^ Scan parquet (50) +- == Initial Plan == - HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (67) + +- Exchange (66) + +- HashAggregate (65) + +- Project (64) + +- Filter (63) + +- Scan parquet (62) -(47) Scan parquet +(50) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(48) NoopFilter +(51) NoopFilter Input [2]: [c_phone#X, c_acctbal#X] Arguments: [c_phone#X, c_acctbal#X] -(49) ProjectExecTransformer +(52) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(50) FlushableHashAggregateExecTransformer +(53) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(51) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(52) ColumnarExchange +(55) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(56) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(54) InputAdapter +(57) InputAdapter Input [2]: [sum#X, count#X] -(55) InputIteratorTransformer +(58) InputIteratorTransformer Input [2]: [sum#X, count#X] -(56) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(58) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(59) Scan parquet +(62) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(60) Filter +(63) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(61) Project +(64) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(62) HashAggregate +(65) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(63) Exchange +(66) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(67) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(65) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt index ee94233a0a5b..8c671a61c9f7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt @@ -1,56 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ NoopFilter (28) + +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (49) - : +- Project (48) - : +- ShuffledHashJoin Inner BuildLeft (47) - : :- Exchange (43) - : : +- Project (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Exchange (53) - +- Project (52) - +- Filter (51) - +- Scan parquet (50) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Project (59) + +- ShuffledHashJoin Inner BuildRight (58) + :- Exchange (53) + : +- Project (52) + : +- ShuffledHashJoin Inner BuildLeft (51) + : :- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Exchange (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -72,248 +76,264 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(23) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(42) Project +(46) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(43) Exchange +(47) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(48) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(45) Filter +(49) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(46) Exchange +(50) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(51) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(48) Project +(52) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Exchange +(53) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(54) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(51) Filter +(55) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(52) Project +(56) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(53) Exchange +(57) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(58) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(55) Project +(59) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(56) HashAggregate +(60) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(57) HashAggregate +(61) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt index 141bd8aa73fd..08e94f8df743 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt @@ -1,47 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftSemi BuildRight (43) - :- Exchange (38) - : +- Project (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftSemi BuildRight (46) + :- Exchange (41) + : +- Project (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -63,202 +66,214 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [l_orderkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [l_orderkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(35) Scan parquet +(38) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(36) Filter +(39) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(37) Project +(40) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(38) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(41) Project +(44) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(42) Exchange +(45) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(44) Project +(47) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(45) HashAggregate +(48) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(46) Exchange +(49) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(50) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(48) Exchange +(51) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(52) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt index 95a5f3ee722f..888a5f38c53d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt @@ -1,115 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (134) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93), Statistics(X) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (133) - +- Exchange (132) - +- HashAggregate (131) - +- Exchange (130) - +- HashAggregate (129) - +- Project (128) - +- ShuffledHashJoin Inner BuildRight (127) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Project (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (126) - +- Project (125) - +- Filter (124) - +- Scan parquet (123) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Project (111) + : : : : +- Filter (110) + : : : : +- Scan parquet (109) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (137) + +- Project (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -131,562 +142,606 @@ Input [2]: [c_custkey#X, c_nationkey#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X + +(66) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: X + +(74) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [1]: [r_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [1]: [r_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(99) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [2]: [n_name#X, revenue#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) Exchange +(108) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(109) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(99) Filter +(110) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(100) Project +(111) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(101) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(103) Project +(114) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(104) Exchange +(115) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(107) Exchange +(118) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(109) Project +(120) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(110) Exchange +(121) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(113) Exchange +(124) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(115) Project +(126) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(116) Exchange +(127) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(119) Exchange +(130) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(121) Project +(132) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(122) Exchange +(133) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(125) Project +(136) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(126) Exchange +(137) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(128) Project +(139) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(129) HashAggregate +(140) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(130) Exchange +(141) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) HashAggregate +(142) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(132) Exchange +(143) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(144) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(134) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt index 5987a808f5fd..fbcc75a247d5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * l_discount#X) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt index 994d1e163a36..a7597d3dbe87 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt @@ -1,110 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (138) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ SortExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ShuffleQueryStage (70), Statistics(X) - +- ReusedExchange (69) + VeloxColumnarToRowExec (100) + +- ^ SortExecTransformer (98) + +- ^ InputIteratorTransformer (97) + +- ShuffleQueryStage (95), Statistics(X) + +- ColumnarExchange (94) + +- VeloxAppendBatches (93) + +- ^ RegularHashAggregateExecTransformer (91) + +- ^ InputIteratorTransformer (90) + +- ShuffleQueryStage (88), Statistics(X) + +- ColumnarExchange (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ReusedExchange (78) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (111) - : : +- Project (110) - : : +- ShuffledHashJoin Inner BuildRight (109) - : : :- Exchange (105) - : : : +- Project (104) - : : : +- ShuffledHashJoin Inner BuildRight (103) - : : : :- Exchange (99) - : : : : +- Project (98) - : : : : +- ShuffledHashJoin Inner BuildLeft (97) - : : : : :- Exchange (93) - : : : : : +- Filter (92) - : : : : : +- Scan parquet (91) - : : : : +- Exchange (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Exchange (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (114) - : +- Filter (113) - : +- Scan parquet (112) - +- Exchange (120) - +- Filter (119) - +- Scan parquet (118) + Sort (137) + +- Exchange (136) + +- HashAggregate (135) + +- Exchange (134) + +- HashAggregate (133) + +- Project (132) + +- ShuffledHashJoin Inner BuildRight (131) + :- Exchange (127) + : +- Project (126) + : +- ShuffledHashJoin Inner BuildRight (125) + : :- Exchange (121) + : : +- Project (120) + : : +- ShuffledHashJoin Inner BuildRight (119) + : : :- Exchange (115) + : : : +- Project (114) + : : : +- ShuffledHashJoin Inner BuildRight (113) + : : : :- Exchange (109) + : : : : +- Project (108) + : : : : +- ShuffledHashJoin Inner BuildLeft (107) + : : : : :- Exchange (103) + : : : : : +- Filter (102) + : : : : : +- Scan parquet (101) + : : : : +- Exchange (106) + : : : : +- Filter (105) + : : : : +- Scan parquet (104) + : : : +- Exchange (112) + : : : +- Filter (111) + : : : +- Scan parquet (110) + : : +- Exchange (118) + : : +- Filter (117) + : : +- Scan parquet (116) + : +- Exchange (124) + : +- Filter (123) + : +- Scan parquet (122) + +- Exchange (130) + +- Filter (129) + +- Scan parquet (128) (1) Scan parquet @@ -126,534 +136,574 @@ Input [2]: [s_suppkey#X, s_nationkey#X] Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(23) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_orderkey#X, o_custkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: X + +(40) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(69) ReusedExchange [Reuses operator id: 58] +(78) ReusedExchange [Reuses operator id: 66] Output [2]: [n_nationkey#X, n_name#X] -(70) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(71) InputAdapter +(80) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(72) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(73) ShuffledHashJoinExecTransformer +(82) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(74) ProjectExecTransformer +(83) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(75) FlushableHashAggregateExecTransformer +(84) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) ProjectExecTransformer +(85) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) WholeStageCodegenTransformer (X) +(86) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(78) ColumnarExchange +(87) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(88) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(80) InputAdapter +(89) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(81) InputIteratorTransformer +(90) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(82) RegularHashAggregateExecTransformer +(91) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(83) WholeStageCodegenTransformer (X) +(92) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(84) ColumnarExchange +(93) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(94) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(95) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(86) InputAdapter +(96) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(87) InputIteratorTransformer +(97) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(88) SortExecTransformer +(98) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(89) WholeStageCodegenTransformer (X) +(99) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(90) VeloxColumnarToRowExec +(100) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(91) Scan parquet +(101) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(102) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(103) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(104) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(105) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Exchange +(106) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(98) Project +(108) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(102) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(104) Project +(114) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(105) Exchange +(115) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(116) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(117) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(118) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(110) Project +(120) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(111) Exchange +(121) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(122) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(113) Filter +(123) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(114) Exchange +(124) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(116) Project +(126) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(117) Exchange +(127) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(128) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(119) Filter +(129) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(120) Exchange +(130) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(122) Project +(132) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(123) HashAggregate +(133) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(124) Exchange +(134) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(135) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(126) Exchange +(136) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(137) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(128) AdaptiveSparkPlan +(138) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt index b483283286d4..2e5a862f832f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt @@ -1,150 +1,165 @@ == Physical Plan == -AdaptiveSparkPlan (177) +AdaptiveSparkPlan (192) +- == Final Plan == - VeloxColumnarToRowExec (125) - +- ^ SortExecTransformer (123) - +- ^ InputIteratorTransformer (122) - +- ShuffleQueryStage (120), Statistics(X) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ShuffleQueryStage (113), Statistics(X) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ShuffleQueryStage (96), Statistics(X) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ShuffleQueryStage (81), Statistics(X) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ShuffleQueryStage (66), Statistics(X) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ShuffleQueryStage (51), Statistics(X) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ShuffleQueryStage (36), Statistics(X) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ShuffleQueryStage (29), Statistics(X) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ NoopFilter (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ShuffleQueryStage (44), Statistics(X) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ NoopFilter (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ShuffleQueryStage (59), Statistics(X) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ NoopFilter (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ShuffleQueryStage (74), Statistics(X) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ NoopFilter (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ NoopFilter (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ShuffleQueryStage (104), Statistics(X) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ NoopFilter (100) - +- ^ Scan parquet (99) + VeloxColumnarToRowExec (140) + +- ^ SortExecTransformer (138) + +- ^ InputIteratorTransformer (137) + +- ShuffleQueryStage (135), Statistics(X) + +- ColumnarExchange (134) + +- VeloxAppendBatches (133) + +- ^ ProjectExecTransformer (131) + +- ^ RegularHashAggregateExecTransformer (130) + +- ^ InputIteratorTransformer (129) + +- ShuffleQueryStage (127), Statistics(X) + +- ColumnarExchange (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109), Statistics(X) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92), Statistics(X) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75), Statistics(X) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58), Statistics(X) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41), Statistics(X) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ NoopFilter (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33), Statistics(X) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ NoopFilter (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50), Statistics(X) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ NoopFilter (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67), Statistics(X) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ NoopFilter (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ NoopFilter (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ NoopFilter (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118), Statistics(X) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ NoopFilter (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (176) - +- Exchange (175) - +- HashAggregate (174) - +- Exchange (173) - +- HashAggregate (172) - +- Project (171) - +- ShuffledHashJoin Inner BuildRight (170) - :- Exchange (165) - : +- Project (164) - : +- ShuffledHashJoin Inner BuildRight (163) - : :- Exchange (159) - : : +- Project (158) - : : +- ShuffledHashJoin Inner BuildRight (157) - : : :- Exchange (153) - : : : +- Project (152) - : : : +- ShuffledHashJoin Inner BuildRight (151) - : : : :- Exchange (147) - : : : : +- Project (146) - : : : : +- ShuffledHashJoin Inner BuildRight (145) - : : : : :- Exchange (141) - : : : : : +- Project (140) - : : : : : +- ShuffledHashJoin Inner BuildRight (139) - : : : : : :- Exchange (135) - : : : : : : +- Project (134) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) - : : : : : : :- Exchange (129) - : : : : : : : +- Project (128) - : : : : : : : +- Filter (127) - : : : : : : : +- Scan parquet (126) - : : : : : : +- Exchange (132) - : : : : : : +- Filter (131) - : : : : : : +- Scan parquet (130) - : : : : : +- Exchange (138) - : : : : : +- Filter (137) - : : : : : +- Scan parquet (136) - : : : : +- Exchange (144) - : : : : +- Filter (143) - : : : : +- Scan parquet (142) - : : : +- Exchange (150) - : : : +- Filter (149) - : : : +- Scan parquet (148) - : : +- Exchange (156) - : : +- Filter (155) - : : +- Scan parquet (154) - : +- Exchange (162) - : +- Filter (161) - : +- Scan parquet (160) - +- Exchange (169) - +- Project (168) - +- Filter (167) - +- Scan parquet (166) + Sort (191) + +- Exchange (190) + +- HashAggregate (189) + +- Exchange (188) + +- HashAggregate (187) + +- Project (186) + +- ShuffledHashJoin Inner BuildRight (185) + :- Exchange (180) + : +- Project (179) + : +- ShuffledHashJoin Inner BuildRight (178) + : :- Exchange (174) + : : +- Project (173) + : : +- ShuffledHashJoin Inner BuildRight (172) + : : :- Exchange (168) + : : : +- Project (167) + : : : +- ShuffledHashJoin Inner BuildRight (166) + : : : :- Exchange (162) + : : : : +- Project (161) + : : : : +- ShuffledHashJoin Inner BuildRight (160) + : : : : :- Exchange (156) + : : : : : +- Project (155) + : : : : : +- ShuffledHashJoin Inner BuildRight (154) + : : : : : :- Exchange (150) + : : : : : : +- Project (149) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (148) + : : : : : : :- Exchange (144) + : : : : : : : +- Project (143) + : : : : : : : +- Filter (142) + : : : : : : : +- Scan parquet (141) + : : : : : : +- Exchange (147) + : : : : : : +- Filter (146) + : : : : : : +- Scan parquet (145) + : : : : : +- Exchange (153) + : : : : : +- Filter (152) + : : : : : +- Scan parquet (151) + : : : : +- Exchange (159) + : : : : +- Filter (158) + : : : : +- Scan parquet (157) + : : : +- Exchange (165) + : : : +- Filter (164) + : : : +- Scan parquet (163) + : : +- Exchange (171) + : : +- Filter (170) + : : +- Scan parquet (169) + : +- Exchange (177) + : +- Filter (176) + : +- Scan parquet (175) + +- Exchange (184) + +- Project (183) + +- Filter (182) + +- Scan parquet (181) (1) Scan parquet @@ -166,746 +181,806 @@ Input [2]: [p_partkey#X, p_type#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: [n_nationkey#X, n_regionkey#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(79) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(80) ColumnarExchange +(90) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: X + +(91) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(92) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(82) InputAdapter +(93) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(83) InputIteratorTransformer +(94) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(84) Scan parquet +(95) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) NoopFilter +(96) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(86) ProjectExecTransformer +(97) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(92) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(93) ProjectExecTransformer +(105) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(94) WholeStageCodegenTransformer (X) +(106) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(95) ColumnarExchange +(107) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: X + +(108) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(109) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(97) InputAdapter +(110) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(98) InputIteratorTransformer +(111) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(99) Scan parquet +(112) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(100) NoopFilter +(113) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(101) ProjectExecTransformer +(114) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(102) WholeStageCodegenTransformer (X) +(115) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(103) ColumnarExchange +(116) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(117) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(104) ShuffleQueryStage +(118) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(105) InputAdapter +(119) InputAdapter Input [1]: [r_regionkey#X] -(106) InputIteratorTransformer +(120) InputIteratorTransformer Input [1]: [r_regionkey#X] -(107) ShuffledHashJoinExecTransformer +(121) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(108) ProjectExecTransformer +(122) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(109) FlushableHashAggregateExecTransformer +(123) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(110) ProjectExecTransformer +(124) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(111) WholeStageCodegenTransformer (X) +(125) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(112) ColumnarExchange +(126) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(113) ShuffleQueryStage +(127) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(114) InputAdapter +(128) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(115) InputIteratorTransformer +(129) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(116) RegularHashAggregateExecTransformer +(130) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(117) ProjectExecTransformer +(131) ProjectExecTransformer Output [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(118) WholeStageCodegenTransformer (X) +(132) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(119) ColumnarExchange +(133) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(134) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(120) ShuffleQueryStage +(135) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(121) InputAdapter +(136) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(122) InputIteratorTransformer +(137) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(123) SortExecTransformer +(138) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(124) WholeStageCodegenTransformer (X) +(139) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(125) VeloxColumnarToRowExec +(140) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(126) Scan parquet +(141) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(127) Filter +(142) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(128) Project +(143) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(129) Exchange +(144) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) Scan parquet +(145) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(131) Filter +(146) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(132) Exchange +(147) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) ShuffledHashJoin +(148) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(134) Project +(149) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(135) Exchange +(150) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Scan parquet +(151) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(137) Filter +(152) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(138) Exchange +(153) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(154) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(140) Project +(155) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(141) Exchange +(156) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) Scan parquet +(157) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(143) Filter +(158) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(144) Exchange +(159) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) ShuffledHashJoin +(160) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(146) Project +(161) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(147) Exchange +(162) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Scan parquet +(163) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(149) Filter +(164) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(150) Exchange +(165) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) ShuffledHashJoin +(166) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(152) Project +(167) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(153) Exchange +(168) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) Scan parquet +(169) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(155) Filter +(170) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(156) Exchange +(171) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) ShuffledHashJoin +(172) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(158) Project +(173) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(159) Exchange +(174) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Scan parquet +(175) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(161) Filter +(176) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(162) Exchange +(177) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) ShuffledHashJoin +(178) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(164) Project +(179) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(165) Exchange +(180) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) Scan parquet +(181) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(167) Filter +(182) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(168) Project +(183) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(169) Exchange +(184) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) ShuffledHashJoin +(185) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(171) Project +(186) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(172) HashAggregate +(187) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(173) Exchange +(188) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(174) HashAggregate +(189) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(175) Exchange +(190) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(191) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(177) AdaptiveSparkPlan +(192) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt index 5d1350564eac..bbc0a7042023 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt @@ -1,114 +1,125 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (144) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93), Statistics(X) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- HashAggregate (130) - +- Exchange (129) - +- HashAggregate (128) - +- Project (127) - +- ShuffledHashJoin Inner BuildRight (126) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (98) - : : : : : +- Project (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (125) - +- Filter (124) - +- Scan parquet (123) + Sort (143) + +- Exchange (142) + +- HashAggregate (141) + +- Exchange (140) + +- HashAggregate (139) + +- Project (138) + +- ShuffledHashJoin Inner BuildRight (137) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (109) + : : : : : +- Project (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -130,558 +141,602 @@ Input [2]: [p_partkey#X, p_name#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: X + +(57) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_orderdate#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: X + +(74) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(99) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(97) Project +(108) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(109) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(110) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(100) Filter +(111) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(101) Exchange +(112) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(103) Project +(114) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(115) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(118) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(109) Project +(120) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(121) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(113) Exchange +(124) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(115) Project +(126) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(116) Exchange +(127) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(119) Exchange +(130) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(121) Project +(132) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(122) Exchange +(133) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(125) Exchange +(136) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(137) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(127) Project +(138) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(128) HashAggregate +(139) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(129) Exchange +(140) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) HashAggregate +(141) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(131) Exchange +(142) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(143) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(133) AdaptiveSparkPlan +(144) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/1.txt index 656f98574483..1c8670c0fea4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt index 313572f951ad..8233c7669206 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt @@ -1,78 +1,84 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- TakeOrderedAndProjectExecTransformer (59) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ShuffleQueryStage (36) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ShuffleQueryStage (21) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ShuffleQueryStage (29) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ FilterExecTransformer (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ShuffleQueryStage (44) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (66) + +- TakeOrderedAndProjectExecTransformer (65) + +- ^ ProjectExecTransformer (63) + +- ^ RegularHashAggregateExecTransformer (62) + +- ^ InputIteratorTransformer (61) + +- ShuffleQueryStage (59) + +- ColumnarExchange (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FilterExecTransformer (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- ShuffledHashJoin Inner BuildRight (81) - :- Exchange (77) - : +- Project (76) - : +- ShuffledHashJoin Inner BuildRight (75) - : :- Exchange (70) - : : +- Project (69) - : : +- ShuffledHashJoin Inner BuildRight (68) - : : :- Exchange (63) - : : : +- Filter (62) - : : : +- Scan parquet (61) - : : +- Exchange (67) - : : +- Project (66) - : : +- Filter (65) - : : +- Scan parquet (64) - : +- Exchange (74) - : +- Project (73) - : +- Filter (72) - : +- Scan parquet (71) - +- Exchange (80) - +- Filter (79) - +- Scan parquet (78) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- ShuffledHashJoin Inner BuildRight (87) + :- Exchange (83) + : +- Project (82) + : +- ShuffledHashJoin Inner BuildRight (81) + : :- Exchange (76) + : : +- Project (75) + : : +- ShuffledHashJoin Inner BuildRight (74) + : : :- Exchange (69) + : : : +- Filter (68) + : : : +- Scan parquet (67) + : : +- Exchange (73) + : : +- Project (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (80) + : +- Project (79) + : +- Filter (78) + : +- Scan parquet (77) + +- Exchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -94,358 +100,382 @@ Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acct Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: X + +(6) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Arguments: ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(55) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(56) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(58) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(59) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(60) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(61) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(62) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(57) ProjectExecTransformer +(63) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(58) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(59) TakeOrderedAndProjectExecTransformer +(65) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(60) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(61) Scan parquet +(67) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) Filter +(68) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(63) Exchange +(69) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(65) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(66) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(67) Exchange +(73) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) ShuffledHashJoin +(74) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(69) Project +(75) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(70) Exchange +(76) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(77) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(72) Filter +(78) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(73) Project +(79) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(74) Exchange +(80) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(81) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(76) Project +(82) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(77) Exchange +(83) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(80) Exchange +(86) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(87) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(88) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(89) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(84) Exchange +(90) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(91) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(86) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt index e91064e3580f..bc664f216eb3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt @@ -1,65 +1,70 @@ == Physical Plan == -AdaptiveSparkPlan (72) +AdaptiveSparkPlan (77) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ SortExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ShuffleQueryStage (38) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (55) + +- ^ SortExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ FilterExecTransformer (46) + +- ^ RegularHashAggregateExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42) + +- ColumnarExchange (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (71) - +- Exchange (70) - +- Filter (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- ShuffledHashJoin Inner BuildRight (64) - :- Exchange (59) - : +- Project (58) - : +- ShuffledHashJoin Inner BuildRight (57) - : :- Exchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- Exchange (56) - : +- Filter (55) - : +- Scan parquet (54) - +- Exchange (63) - +- Project (62) - +- Filter (61) - +- Scan parquet (60) + Sort (76) + +- Exchange (75) + +- Filter (74) + +- HashAggregate (73) + +- Exchange (72) + +- HashAggregate (71) + +- Project (70) + +- ShuffledHashJoin Inner BuildRight (69) + :- Exchange (64) + : +- Project (63) + : +- ShuffledHashJoin Inner BuildRight (62) + : :- Exchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- Exchange (61) + : +- Filter (60) + : +- Scan parquet (59) + +- Exchange (68) + +- Project (67) + +- Filter (66) + +- Scan parquet (65) (1) Scan parquet @@ -81,292 +86,312 @@ Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [1]: [n_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [n_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(34) FlushableHashAggregateExecTransformer +(38) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(41) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(43) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(45) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(42) FilterExecTransformer +(46) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(43) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(44) ColumnarExchange +(48) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(49) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(46) InputAdapter +(51) InputAdapter Input [2]: [ps_partkey#X, value#X] -(47) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(48) SortExecTransformer +(53) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(49) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(50) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(51) Scan parquet +(56) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(52) Filter +(57) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(53) Exchange +(58) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(59) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(60) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) Exchange +(61) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(62) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(58) Project +(63) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(59) Exchange +(64) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(65) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(61) Filter +(66) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(62) Project +(67) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(63) Exchange +(68) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(69) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(65) Project +(70) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(66) HashAggregate +(71) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(67) Exchange +(72) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(73) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(69) Filter +(74) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(70) Exchange +(75) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(76) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(72) AdaptiveSparkPlan +(77) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt index 8bad82473a58..2004f617a1d8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt @@ -1,46 +1,49 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin Inner BuildLeft (42) - :- Exchange (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- ShuffledHashJoin Inner BuildLeft (45) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -62,196 +65,208 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Arguments: ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(31) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) Scan parquet +(38) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(36) Filter +(39) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(37) Exchange +(40) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(41) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(40) Project +(43) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(41) Exchange +(44) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(45) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(43) Project +(46) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(44) HashAggregate +(47) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(45) Exchange +(48) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(49) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(47) Exchange +(50) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(51) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt index 3268390701fc..0e57911ad3ca 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt @@ -1,49 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ShuffleQueryStage (5) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftOuter BuildRight (43) - :- Exchange (38) - : +- Scan parquet (37) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftOuter BuildRight (46) + :- Exchange (41) + : +- Scan parquet (40) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -60,223 +63,235 @@ Input [1]: [c_custkey#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(4) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) FilterExecTransformer +(10) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Arguments: ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(44) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) Exchange +(45) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(47) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(48) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt index dd4a53cfa822..2bc0be8fcb67 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (23) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (25) + +- ^ ProjectExecTransformer (23) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (34) - +- HashAggregate (33) - +- Project (32) - +- ShuffledHashJoin Inner BuildRight (31) - :- Exchange (27) - : +- Project (26) - : +- Filter (25) - : +- Scan parquet (24) - +- Exchange (30) - +- Filter (29) - +- Scan parquet (28) + HashAggregate (36) + +- HashAggregate (35) + +- Project (34) + +- ShuffledHashJoin Inner BuildRight (33) + :- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -52,144 +54,152 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [2]: [p_partkey#X, p_type#X] Arguments: isnotnull(p_partkey#X) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(21) ProjectExecTransformer +(23) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(22) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(23) VeloxColumnarToRowExec +(25) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(24) Scan parquet +(26) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(26) Project +(28) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) Exchange +(29) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Scan parquet +(30) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(30) Exchange +(32) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) ShuffledHashJoin +(33) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(32) Project +(34) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(34) HashAggregate +(36) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] -(35) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt index c456274bfeaa..40bb7d7d7983 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt @@ -1,44 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (47) +- == Final Plan == - VeloxColumnarToRowExec (30) - +- ^ SortExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (32) + +- ^ SortExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (22) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- Project (42) - +- ShuffledHashJoin Inner BuildLeft (41) - :- Exchange (33) - : +- Filter (32) - : +- Scan parquet (31) - +- Filter (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- Filter (35) - +- Scan parquet (34) + Sort (46) + +- Exchange (45) + +- Project (44) + +- ShuffledHashJoin Inner BuildLeft (43) + :- Exchange (35) + : +- Filter (34) + : +- Scan parquet (33) + +- Filter (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- Filter (37) + +- Scan parquet (36) (1) Scan parquet @@ -60,182 +62,190 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(16) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(20) FilterExecTransformer +(21) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(21) ShuffledHashJoinExecTransformer +(22) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(26) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) SortExecTransformer +(30) SortExecTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(30) VeloxColumnarToRowExec +(32) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(31) Scan parquet +(33) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(32) Filter +(34) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(33) Exchange +(35) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) Scan parquet +(36) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(35) Filter +(37) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(36) Project +(38) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(37) HashAggregate +(39) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(38) Exchange +(40) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(41) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(40) Filter +(42) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(41) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(42) Project +(44) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(43) Exchange +(45) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(46) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(47) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt index 7bbe4249fdcd..90afc1a3db17 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt @@ -1,59 +1,62 @@ == Physical Plan == -AdaptiveSparkPlan (64) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (42) - +- ^ SortExecTransformer (40) - +- ^ InputIteratorTransformer (39) - +- ShuffleQueryStage (37) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (45) + +- ^ SortExecTransformer (43) + +- ^ InputIteratorTransformer (42) + +- ShuffleQueryStage (40) + +- ColumnarExchange (39) + +- VeloxAppendBatches (38) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- ^ ProjectExecTransformer (30) + +- ^ FlushableHashAggregateExecTransformer (29) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (63) - +- Exchange (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- HashAggregate (58) - +- Exchange (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (50) - : +- BroadcastHashJoin LeftAnti BuildRight (49) - : :- Filter (44) - : : +- Scan parquet (43) - : +- BroadcastExchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (53) - +- Filter (52) - +- Scan parquet (51) + Sort (66) + +- Exchange (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- HashAggregate (61) + +- Exchange (60) + +- HashAggregate (59) + +- Project (58) + +- ShuffledHashJoin Inner BuildRight (57) + :- Exchange (53) + : +- BroadcastHashJoin LeftAnti BuildRight (52) + : :- Filter (47) + : : +- Scan parquet (46) + : +- BroadcastExchange (51) + : +- Project (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -75,270 +78,282 @@ Input [2]: [ps_partkey#X, ps_suppkey#X] Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) FlushableHashAggregateExecTransformer +(29) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(30) ColumnarExchange +(32) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(32) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) ColumnarExchange +(38) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(39) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(40) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(38) InputAdapter +(41) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) InputIteratorTransformer +(42) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(40) SortExecTransformer +(43) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(41) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(42) VeloxColumnarToRowExec +(45) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) Scan parquet +(46) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(45) Scan parquet +(48) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(46) Filter +(49) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(47) Project +(50) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(48) BroadcastExchange +(51) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(49) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(50) Exchange +(53) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(54) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(53) Exchange +(56) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(57) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(55) Project +(58) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(56) HashAggregate +(59) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) HashAggregate +(61) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(59) HashAggregate +(62) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(60) Exchange +(63) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(62) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) Sort +(66) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(64) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt index 248f638db9ad..04753848cb48 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt @@ -1,56 +1,58 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ ProjectExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) - :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ FilterExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ FilterExecTransformer (20) - +- ^ Scan parquet (19) + VeloxColumnarToRowExec (39) + +- ^ ProjectExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ ProjectExecTransformer (34) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (33) + :- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ FilterExecTransformer (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (56) - +- HashAggregate (55) - +- Project (54) - +- ShuffledHashJoin Inner BuildRight (53) - :- Project (46) - : +- ShuffledHashJoin Inner BuildRight (45) - : :- Exchange (40) - : : +- Filter (39) - : : +- Scan parquet (38) - : +- Exchange (44) - : +- Project (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Filter (48) - +- Scan parquet (47) + HashAggregate (58) + +- HashAggregate (57) + +- Project (56) + +- ShuffledHashJoin Inner BuildRight (55) + :- Project (48) + : +- ShuffledHashJoin Inner BuildRight (47) + : :- Exchange (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Exchange (46) + : +- Project (45) + : +- Filter (44) + : +- Scan parquet (43) + +- Filter (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Filter (50) + +- Scan parquet (49) (1) Scan parquet @@ -72,250 +74,258 @@ Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [p_partkey#X, p_brand#X, p_container#X] Arguments: ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(19) Scan parquet +(21) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(20) FilterExecTransformer +(22) FilterExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Arguments: isnotnull(l_partkey#X) -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(30) FilterExecTransformer +(32) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(31) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(32) ProjectExecTransformer +(34) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(33) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(35) ProjectExecTransformer +(37) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(36) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(37) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(38) Scan parquet +(40) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(39) Filter +(41) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(40) Exchange +(42) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(43) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(43) Project +(45) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(44) Exchange +(46) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(46) Project +(48) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(47) Scan parquet +(49) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(48) Filter +(50) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(49) HashAggregate +(51) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(50) Exchange +(52) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) Filter +(54) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(53) ShuffledHashJoin +(55) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(54) Project +(56) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) HashAggregate +(57) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(56) HashAggregate +(58) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] -(57) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt index 428186218984..516601a5f469 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt @@ -1,89 +1,94 @@ == Physical Plan == -AdaptiveSparkPlan (97) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- TakeOrderedAndProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) - :- ^ InputIteratorTransformer (41) - : +- ShuffleQueryStage (39) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ShuffleQueryStage (32) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ShuffleQueryStage (22) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) - :- ^ InputIteratorTransformer (49) - : +- ShuffleQueryStage (47) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ FilterExecTransformer (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ShuffleQueryStage (51) - +- ReusedExchange (50) + VeloxColumnarToRowExec (69) + +- TakeOrderedAndProjectExecTransformer (68) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ RegularHashAggregateExecTransformer (65) + +- ^ ProjectExecTransformer (64) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (63) + :- ^ InputIteratorTransformer (45) + : +- ShuffleQueryStage (43) + : +- ColumnarExchange (42) + : +- VeloxAppendBatches (41) + : +- ^ ProjectExecTransformer (39) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (38) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (37) + : +- ShuffleQueryStage (35) + : +- ColumnarExchange (34) + : +- VeloxAppendBatches (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (30) + : :- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ Scan parquet (10) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ RegularHashAggregateExecTransformer (27) + : +- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24) + : +- ColumnarExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) + :- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52) + : +- ColumnarExchange (51) + : +- VeloxAppendBatches (50) + : +- ^ ProjectExecTransformer (48) + : +- ^ FilterExecTransformer (47) + : +- ^ Scan parquet (46) + +- ^ ProjectExecTransformer (61) + +- ^ FilterExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56) + +- ReusedExchange (55) +- == Initial Plan == - TakeOrderedAndProject (96) - +- HashAggregate (95) - +- HashAggregate (94) - +- Project (93) - +- ShuffledHashJoin Inner BuildRight (92) - :- Exchange (81) - : +- Project (80) - : +- ShuffledHashJoin Inner BuildLeft (79) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (78) - : +- ShuffledHashJoin LeftSemi BuildRight (77) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Project (76) - : +- Filter (75) - : +- HashAggregate (74) - : +- Exchange (73) - : +- HashAggregate (72) - : +- Scan parquet (71) - +- ShuffledHashJoin LeftSemi BuildRight (91) - :- Exchange (84) - : +- Filter (83) - : +- Scan parquet (82) - +- Project (90) - +- Filter (89) - +- HashAggregate (88) - +- Exchange (87) - +- HashAggregate (86) - +- Scan parquet (85) + TakeOrderedAndProject (101) + +- HashAggregate (100) + +- HashAggregate (99) + +- Project (98) + +- ShuffledHashJoin Inner BuildRight (97) + :- Exchange (86) + : +- Project (85) + : +- ShuffledHashJoin Inner BuildLeft (84) + : :- Exchange (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (83) + : +- ShuffledHashJoin LeftSemi BuildRight (82) + : :- Exchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- Project (81) + : +- Filter (80) + : +- HashAggregate (79) + : +- Exchange (78) + : +- HashAggregate (77) + : +- Scan parquet (76) + +- ShuffledHashJoin LeftSemi BuildRight (96) + :- Exchange (89) + : +- Filter (88) + : +- Scan parquet (87) + +- Project (95) + +- Filter (94) + +- HashAggregate (93) + +- Exchange (92) + +- HashAggregate (91) + +- Scan parquet (90) (1) Scan parquet @@ -105,420 +110,440 @@ Input [2]: [c_custkey#X, c_name#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(26) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(27) ProjectExecTransformer +(29) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(28) ShuffledHashJoinExecTransformer +(30) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(30) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(31) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(33) InputAdapter +(36) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(34) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) ShuffledHashJoinExecTransformer +(38) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(36) ProjectExecTransformer +(39) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(38) ColumnarExchange +(41) VeloxAppendBatches +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(42) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(43) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(40) InputAdapter +(44) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(41) InputIteratorTransformer +(45) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(42) Scan parquet +(46) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(43) FilterExecTransformer +(47) FilterExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Arguments: isnotnull(l_orderkey#X) -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(46) ColumnarExchange +(50) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: X + +(51) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(52) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(48) InputAdapter +(53) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(49) InputIteratorTransformer +(54) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(50) ReusedExchange [Reuses operator id: 21] +(55) ReusedExchange [Reuses operator id: 23] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) ShuffleQueryStage +(56) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(52) InputAdapter +(57) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(53) InputIteratorTransformer +(58) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(54) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(55) FilterExecTransformer +(60) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(56) ProjectExecTransformer +(61) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(58) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(59) ProjectExecTransformer +(64) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(60) RegularHashAggregateExecTransformer +(65) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(61) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(62) WholeStageCodegenTransformer (X) +(67) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(63) TakeOrderedAndProjectExecTransformer +(68) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(64) VeloxColumnarToRowExec +(69) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(65) Scan parquet +(70) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(66) Filter +(71) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(67) Exchange +(72) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(73) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(69) Filter +(74) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(70) Exchange +(75) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(76) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(77) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(78) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(79) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(80) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(81) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Exchange +(83) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) ShuffledHashJoin +(84) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(80) Project +(85) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(81) Exchange +(86) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(87) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(83) Filter +(88) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(84) Exchange +(89) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(90) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(86) HashAggregate +(91) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(87) Exchange +(92) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) HashAggregate +(93) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(89) Filter +(94) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(90) Project +(95) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(91) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(93) Project +(98) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(94) HashAggregate +(99) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(95) HashAggregate +(100) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(96) TakeOrderedAndProject +(101) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(97) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt index 4ea9d09e13f7..d3d74c5ba792 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (33) - +- HashAggregate (32) - +- Project (31) - +- ShuffledHashJoin Inner BuildRight (30) - :- Exchange (26) - : +- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- Exchange (29) - +- Filter (28) - +- Scan parquet (27) + HashAggregate (35) + +- HashAggregate (34) + +- Project (33) + +- ShuffledHashJoin Inner BuildRight (32) + :- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) (1) Scan parquet @@ -51,140 +53,148 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(24) VeloxColumnarToRowExec Input [1]: [revenue#X] -(23) Scan parquet +(25) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(24) Filter +(26) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(25) Project +(27) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(26) Exchange +(28) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Scan parquet +(29) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(28) Filter +(30) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(29) Exchange +(31) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) ShuffledHashJoin +(32) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(31) Project +(33) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(32) HashAggregate +(34) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(34) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt index 4fe956080086..df1ae98f903e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt @@ -1,110 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (126) +AdaptiveSparkPlan (136) +- == Final Plan == - VeloxColumnarToRowExec (86) - +- ^ SortExecTransformer (84) - +- ^ InputIteratorTransformer (83) - +- ShuffleQueryStage (81) - +- ColumnarExchange (80) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ShuffleQueryStage (29) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ShuffleQueryStage (14) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ FilterExecTransformer (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ShuffleQueryStage (22) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ FilterExecTransformer (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ FilterExecTransformer (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41) - : +- ReusedExchange (40) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (96) + +- ^ SortExecTransformer (94) + +- ^ InputIteratorTransformer (93) + +- ShuffleQueryStage (91) + +- ColumnarExchange (90) + +- VeloxAppendBatches (89) + +- ^ ProjectExecTransformer (87) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (86) + :- ^ InputIteratorTransformer (76) + : +- ShuffleQueryStage (74) + : +- ColumnarExchange (73) + : +- VeloxAppendBatches (72) + : +- ^ ProjectExecTransformer (70) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (69) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- VeloxAppendBatches (64) + : +- ^ ProjectExecTransformer (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (61) + : :- ^ InputIteratorTransformer (35) + : : +- ShuffleQueryStage (33) + : : +- ColumnarExchange (32) + : : +- VeloxAppendBatches (31) + : : +- ^ ProjectExecTransformer (29) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : : :- ^ InputIteratorTransformer (18) + : : : +- ShuffleQueryStage (16) + : : : +- ColumnarExchange (15) + : : : +- VeloxAppendBatches (14) + : : : +- ^ ProjectExecTransformer (12) + : : : +- ^ FilterExecTransformer (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (27) + : : +- ShuffleQueryStage (25) + : : +- ColumnarExchange (24) + : : +- VeloxAppendBatches (23) + : : +- ^ ProjectExecTransformer (21) + : : +- ^ FilterExecTransformer (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (60) + : +- ShuffleQueryStage (58) + : +- ColumnarExchange (57) + : +- VeloxAppendBatches (56) + : +- ^ ProjectExecTransformer (54) + : +- ^ FilterExecTransformer (53) + : +- ^ ProjectExecTransformer (52) + : +- ^ RegularHashAggregateExecTransformer (51) + : +- ^ RegularHashAggregateExecTransformer (50) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (49) + : :- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ FilterExecTransformer (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (48) + : +- ShuffleQueryStage (46) + : +- ReusedExchange (45) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83) + +- ColumnarExchange (82) + +- VeloxAppendBatches (81) + +- ^ ProjectExecTransformer (79) + +- ^ FilterExecTransformer (78) + +- ^ Scan parquet (77) +- == Initial Plan == - Sort (125) - +- Exchange (124) - +- Project (123) - +- ShuffledHashJoin Inner BuildRight (122) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin LeftSemi BuildRight (115) - : :- Exchange (89) - : : +- Filter (88) - : : +- Scan parquet (87) - : +- Exchange (114) - : +- Project (113) - : +- ShuffledHashJoin Inner BuildLeft (112) - : :- Exchange (98) - : : +- ShuffledHashJoin LeftSemi BuildRight (97) - : : :- Exchange (92) - : : : +- Filter (91) - : : : +- Scan parquet (90) - : : +- Exchange (96) - : : +- Project (95) - : : +- Filter (94) - : : +- Scan parquet (93) - : +- Exchange (111) - : +- Filter (110) - : +- HashAggregate (109) - : +- HashAggregate (108) - : +- ShuffledHashJoin LeftSemi BuildRight (107) - : :- Exchange (102) - : : +- Project (101) - : : +- Filter (100) - : : +- Scan parquet (99) - : +- Exchange (106) - : +- Project (105) - : +- Filter (104) - : +- Scan parquet (103) - +- Exchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (135) + +- Exchange (134) + +- Project (133) + +- ShuffledHashJoin Inner BuildRight (132) + :- Exchange (127) + : +- Project (126) + : +- ShuffledHashJoin LeftSemi BuildRight (125) + : :- Exchange (99) + : : +- Filter (98) + : : +- Scan parquet (97) + : +- Exchange (124) + : +- Project (123) + : +- ShuffledHashJoin Inner BuildLeft (122) + : :- Exchange (108) + : : +- ShuffledHashJoin LeftSemi BuildRight (107) + : : :- Exchange (102) + : : : +- Filter (101) + : : : +- Scan parquet (100) + : : +- Exchange (106) + : : +- Project (105) + : : +- Filter (104) + : : +- Scan parquet (103) + : +- Exchange (121) + : +- Filter (120) + : +- HashAggregate (119) + : +- HashAggregate (118) + : +- ShuffledHashJoin LeftSemi BuildRight (117) + : :- Exchange (112) + : : +- Project (111) + : : +- Filter (110) + : : +- Scan parquet (109) + : +- Exchange (116) + : +- Project (115) + : +- Filter (114) + : +- Scan parquet (113) + +- Exchange (131) + +- Project (130) + +- Filter (129) + +- Scan parquet (128) (1) Scan parquet @@ -126,518 +136,558 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(18) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [p_partkey#X, p_name#X] Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [1]: [p_partkey#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [1]: [p_partkey#X] -(25) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(32) Scan parquet +(36) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(33) FilterExecTransformer +(37) FilterExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(40) ReusedExchange [Reuses operator id: 21] +(45) ReusedExchange [Reuses operator id: 24] Output [1]: [p_partkey#X] -(41) ShuffleQueryStage +(46) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(42) InputAdapter +(47) InputAdapter Input [1]: [p_partkey#X] -(43) InputIteratorTransformer +(48) InputIteratorTransformer Input [1]: [p_partkey#X] -(44) ShuffledHashJoinExecTransformer +(49) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(45) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(46) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(47) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(48) FilterExecTransformer +(53) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(49) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(51) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(53) InputAdapter +(59) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(54) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(55) ShuffledHashJoinExecTransformer +(61) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(58) ColumnarExchange +(64) VeloxAppendBatches +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: X + +(65) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(66) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(60) InputAdapter +(67) InputAdapter Input [1]: [ps_suppkey#X] -(61) InputIteratorTransformer +(68) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(62) ShuffledHashJoinExecTransformer +(69) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(63) ProjectExecTransformer +(70) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(71) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(72) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(73) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(74) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(75) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(68) InputIteratorTransformer +(76) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(69) Scan parquet +(77) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer +(78) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(71) ProjectExecTransformer +(79) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(73) ColumnarExchange +(81) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(82) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(83) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(75) InputAdapter +(84) InputAdapter Input [1]: [n_nationkey#X] -(76) InputIteratorTransformer +(85) InputIteratorTransformer Input [1]: [n_nationkey#X] -(77) ShuffledHashJoinExecTransformer +(86) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(87) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(79) WholeStageCodegenTransformer (X) +(88) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(80) ColumnarExchange +(89) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(90) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(91) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(82) InputAdapter +(92) InputAdapter Input [2]: [s_name#X, s_address#X] -(83) InputIteratorTransformer +(93) InputIteratorTransformer Input [2]: [s_name#X, s_address#X] -(84) SortExecTransformer +(94) SortExecTransformer Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(85) WholeStageCodegenTransformer (X) +(95) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(86) VeloxColumnarToRowExec +(96) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(87) Scan parquet +(97) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(88) Filter +(98) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(89) Exchange +(99) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(100) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(91) Filter +(101) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(92) Exchange +(102) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Scan parquet +(103) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(94) Filter +(104) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(95) Project +(105) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(96) Exchange +(106) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(98) Exchange +(108) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(109) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(100) Filter +(110) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(101) Project +(111) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(102) Exchange +(112) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) Scan parquet +(113) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(104) Filter +(114) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(105) Project +(115) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(106) Exchange +(116) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(117) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(108) HashAggregate +(118) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(109) HashAggregate +(119) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(110) Filter +(120) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(111) Exchange +(121) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(113) Project +(123) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(114) Exchange +(124) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(116) Project +(126) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(117) Exchange +(127) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(128) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(119) Filter +(129) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(120) Project +(130) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(121) Exchange +(131) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(123) Project +(133) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(124) Exchange +(134) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) Sort +(135) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(126) AdaptiveSparkPlan +(136) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt index 6f94bd5c8d01..664bb4e88905 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt @@ -1,104 +1,113 @@ == Physical Plan == -AdaptiveSparkPlan (119) +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (82) - +- TakeOrderedAndProjectExecTransformer (81) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ShuffleQueryStage (21) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ShuffleQueryStage (30) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ FilterExecTransformer (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ShuffleQueryStage (67) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ FilterExecTransformer (63) - +- ^ Scan parquet (62) + VeloxColumnarToRowExec (91) + +- TakeOrderedAndProjectExecTransformer (90) + +- ^ RegularHashAggregateExecTransformer (88) + +- ^ InputIteratorTransformer (87) + +- ShuffleQueryStage (85) + +- ColumnarExchange (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ FilterExecTransformer (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ FilterExecTransformer (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ FilterExecTransformer (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (118) - +- HashAggregate (117) - +- Exchange (116) - +- HashAggregate (115) - +- Project (114) - +- ShuffledHashJoin Inner BuildRight (113) - :- Exchange (108) - : +- Project (107) - : +- ShuffledHashJoin Inner BuildRight (106) - : :- Exchange (101) - : : +- Project (100) - : : +- ShuffledHashJoin Inner BuildLeft (99) - : : :- Exchange (85) - : : : +- Filter (84) - : : : +- Scan parquet (83) - : : +- Exchange (98) - : : +- ShuffledHashJoin LeftAnti BuildRight (97) - : : :- ShuffledHashJoin LeftSemi BuildRight (92) - : : : :- Exchange (89) - : : : : +- Project (88) - : : : : +- Filter (87) - : : : : +- Scan parquet (86) - : : : +- Exchange (91) - : : : +- Scan parquet (90) - : : +- Exchange (96) - : : +- Project (95) - : : +- Filter (94) - : : +- Scan parquet (93) - : +- Exchange (105) - : +- Project (104) - : +- Filter (103) - : +- Scan parquet (102) - +- Exchange (112) - +- Project (111) - +- Filter (110) - +- Scan parquet (109) + TakeOrderedAndProject (127) + +- HashAggregate (126) + +- Exchange (125) + +- HashAggregate (124) + +- Project (123) + +- ShuffledHashJoin Inner BuildRight (122) + :- Exchange (117) + : +- Project (116) + : +- ShuffledHashJoin Inner BuildRight (115) + : :- Exchange (110) + : : +- Project (109) + : : +- ShuffledHashJoin Inner BuildLeft (108) + : : :- Exchange (94) + : : : +- Filter (93) + : : : +- Scan parquet (92) + : : +- Exchange (107) + : : +- ShuffledHashJoin LeftAnti BuildRight (106) + : : :- ShuffledHashJoin LeftSemi BuildRight (101) + : : : :- Exchange (98) + : : : : +- Project (97) + : : : : +- Filter (96) + : : : : +- Scan parquet (95) + : : : +- Exchange (100) + : : : +- Scan parquet (99) + : : +- Exchange (105) + : : +- Project (104) + : : +- Filter (103) + : : +- Scan parquet (102) + : +- Exchange (114) + : +- Project (113) + : +- Filter (112) + : +- Scan parquet (111) + +- Exchange (121) + +- Project (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -120,494 +129,530 @@ Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(24) ShuffledHashJoinExecTransformer +(27) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(25) Scan parquet +(28) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(26) FilterExecTransformer +(29) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(29) ColumnarExchange +(32) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(33) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(33) ShuffledHashJoinExecTransformer +(37) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(41) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(40) ShuffledHashJoinExecTransformer +(45) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(47) Scan parquet +(53) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(48) FilterExecTransformer +(54) FilterExecTransformer Input [2]: [o_orderkey#X, o_orderstatus#X] Arguments: ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(49) ProjectExecTransformer +(55) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(50) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(51) ColumnarExchange +(57) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: X + +(58) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(53) InputAdapter +(60) InputAdapter Input [1]: [o_orderkey#X] -(54) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [o_orderkey#X] -(55) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(62) Scan parquet +(70) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(63) FilterExecTransformer +(71) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(64) ProjectExecTransformer +(72) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(65) WholeStageCodegenTransformer (X) +(73) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(66) ColumnarExchange +(74) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(75) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(76) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(68) InputAdapter +(77) InputAdapter Input [1]: [n_nationkey#X] -(69) InputIteratorTransformer +(78) InputIteratorTransformer Input [1]: [n_nationkey#X] -(70) ShuffledHashJoinExecTransformer +(79) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(72) FlushableHashAggregateExecTransformer +(81) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(73) ProjectExecTransformer +(82) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(74) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(75) ColumnarExchange +(84) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(85) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(77) InputAdapter +(86) InputAdapter Input [2]: [s_name#X, count#X] -(78) InputIteratorTransformer +(87) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(79) RegularHashAggregateExecTransformer +(88) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(80) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(81) TakeOrderedAndProjectExecTransformer +(90) TakeOrderedAndProjectExecTransformer Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X], 0 -(82) VeloxColumnarToRowExec +(91) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(83) Scan parquet +(92) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(84) Filter +(93) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(85) Exchange +(94) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(86) Scan parquet +(95) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(87) Filter +(96) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(88) Project +(97) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(89) Exchange +(98) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(99) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(91) Exchange +(100) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) ShuffledHashJoin +(101) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(93) Scan parquet +(102) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(94) Filter +(103) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(95) Project +(104) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(96) Exchange +(105) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(106) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(98) Exchange +(107) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(100) Project +(109) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(101) Exchange +(110) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(103) Filter +(112) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(104) Project +(113) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(105) Exchange +(114) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) ShuffledHashJoin +(115) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(107) Project +(116) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(108) Exchange +(117) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Scan parquet +(118) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(110) Filter +(119) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(111) Project +(120) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(112) Exchange +(121) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(114) Project +(123) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(115) HashAggregate +(124) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(116) Exchange +(125) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) HashAggregate +(126) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(118) TakeOrderedAndProject +(127) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(119) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt index 77c3b584125b..5ada352ba19c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt @@ -1,43 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (46) +AdaptiveSparkPlan (49) +- == Final Plan == - VeloxColumnarToRowExec (33) - +- ^ SortExecTransformer (31) - +- ^ InputIteratorTransformer (30) - +- ShuffleQueryStage (28) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (45) - +- Exchange (44) - +- HashAggregate (43) - +- Exchange (42) - +- HashAggregate (41) - +- Project (40) - +- ShuffledHashJoin LeftAnti BuildRight (39) - :- Exchange (36) - : +- Filter (35) - : +- Scan parquet (34) - +- Exchange (38) - +- Scan parquet (37) + Sort (48) + +- Exchange (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- ShuffledHashJoin LeftAnti BuildRight (42) + :- Exchange (39) + : +- Filter (38) + : +- Scan parquet (37) + +- Exchange (41) + +- Scan parquet (40) (1) Scan parquet @@ -59,182 +62,194 @@ Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) Scan parquet +(10) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [1]: [o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [1]: [o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(26) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) ColumnarExchange +(29) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(30) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(28) ShuffleQueryStage +(31) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(29) InputAdapter +(32) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) InputIteratorTransformer +(33) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(31) SortExecTransformer +(34) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(32) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(33) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) Scan parquet +(37) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(35) Filter +(38) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(36) Exchange +(39) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(40) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(40) Project +(43) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(41) HashAggregate +(44) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(42) Exchange +(45) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) HashAggregate +(46) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(44) Exchange +(47) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(48) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(46) AdaptiveSparkPlan +(49) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt index 6e4f9178de19..c51701bd0840 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt @@ -1,56 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (49) - : +- Project (48) - : +- ShuffledHashJoin Inner BuildLeft (47) - : :- Exchange (43) - : : +- Project (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Exchange (53) - +- Project (52) - +- Filter (51) - +- Scan parquet (50) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Project (59) + +- ShuffledHashJoin Inner BuildRight (58) + :- Exchange (53) + : +- Project (52) + : +- ShuffledHashJoin Inner BuildLeft (51) + : :- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Exchange (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -72,244 +76,260 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(23) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(42) Project +(46) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(43) Exchange +(47) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(48) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(45) Filter +(49) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(46) Exchange +(50) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(51) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(48) Project +(52) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Exchange +(53) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(54) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(51) Filter +(55) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(52) Project +(56) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(53) Exchange +(57) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(58) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(55) Project +(59) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(56) HashAggregate +(60) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(57) HashAggregate +(61) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt index d2e7a9cffbd6..2f1dd53da1ef 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt @@ -1,47 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftSemi BuildRight (43) - :- Exchange (38) - : +- Project (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftSemi BuildRight (46) + :- Exchange (41) + : +- Project (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -63,200 +66,212 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [l_orderkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [l_orderkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(35) Scan parquet +(38) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(36) Filter +(39) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(37) Project +(40) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(38) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(41) Project +(44) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(42) Exchange +(45) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(44) Project +(47) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(45) HashAggregate +(48) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(46) Exchange +(49) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(50) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(48) Exchange +(51) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(52) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt index aff82bdec961..e20b31dd7907 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt @@ -1,115 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (134) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (133) - +- Exchange (132) - +- HashAggregate (131) - +- Exchange (130) - +- HashAggregate (129) - +- Project (128) - +- ShuffledHashJoin Inner BuildRight (127) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Project (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (126) - +- Project (125) - +- Filter (124) - +- Scan parquet (123) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Project (111) + : : : : +- Filter (110) + : : : : +- Scan parquet (109) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (137) + +- Project (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -131,552 +142,596 @@ Input [2]: [c_custkey#X, c_nationkey#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X + +(66) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: X + +(74) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [r_regionkey#X, r_name#X] Arguments: ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [1]: [r_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [1]: [r_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(99) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [2]: [n_name#X, revenue#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) Exchange +(108) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(109) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(99) Filter +(110) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(100) Project +(111) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(101) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(103) Project +(114) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(104) Exchange +(115) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(107) Exchange +(118) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(109) Project +(120) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(110) Exchange +(121) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(113) Exchange +(124) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(115) Project +(126) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(116) Exchange +(127) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(119) Exchange +(130) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(121) Project +(132) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(122) Exchange +(133) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(125) Project +(136) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(126) Exchange +(137) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(128) Project +(139) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(129) HashAggregate +(140) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(130) Exchange +(141) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) HashAggregate +(142) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(132) Exchange +(143) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(144) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(134) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/6.txt index b39d8c1b2aec..41f69e74d1a0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnot Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt index f0650d65e2cf..e7c225796976 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt @@ -1,110 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (138) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ SortExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ShuffleQueryStage (70) - +- ReusedExchange (69) + VeloxColumnarToRowExec (100) + +- ^ SortExecTransformer (98) + +- ^ InputIteratorTransformer (97) + +- ShuffleQueryStage (95) + +- ColumnarExchange (94) + +- VeloxAppendBatches (93) + +- ^ RegularHashAggregateExecTransformer (91) + +- ^ InputIteratorTransformer (90) + +- ShuffleQueryStage (88) + +- ColumnarExchange (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79) + +- ReusedExchange (78) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (111) - : : +- Project (110) - : : +- ShuffledHashJoin Inner BuildRight (109) - : : :- Exchange (105) - : : : +- Project (104) - : : : +- ShuffledHashJoin Inner BuildRight (103) - : : : :- Exchange (99) - : : : : +- Project (98) - : : : : +- ShuffledHashJoin Inner BuildLeft (97) - : : : : :- Exchange (93) - : : : : : +- Filter (92) - : : : : : +- Scan parquet (91) - : : : : +- Exchange (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Exchange (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (114) - : +- Filter (113) - : +- Scan parquet (112) - +- Exchange (120) - +- Filter (119) - +- Scan parquet (118) + Sort (137) + +- Exchange (136) + +- HashAggregate (135) + +- Exchange (134) + +- HashAggregate (133) + +- Project (132) + +- ShuffledHashJoin Inner BuildRight (131) + :- Exchange (127) + : +- Project (126) + : +- ShuffledHashJoin Inner BuildRight (125) + : :- Exchange (121) + : : +- Project (120) + : : +- ShuffledHashJoin Inner BuildRight (119) + : : :- Exchange (115) + : : : +- Project (114) + : : : +- ShuffledHashJoin Inner BuildRight (113) + : : : :- Exchange (109) + : : : : +- Project (108) + : : : : +- ShuffledHashJoin Inner BuildLeft (107) + : : : : :- Exchange (103) + : : : : : +- Filter (102) + : : : : : +- Scan parquet (101) + : : : : +- Exchange (106) + : : : : +- Filter (105) + : : : : +- Scan parquet (104) + : : : +- Exchange (112) + : : : +- Filter (111) + : : : +- Scan parquet (110) + : : +- Exchange (118) + : : +- Filter (117) + : : +- Scan parquet (116) + : +- Exchange (124) + : +- Filter (123) + : +- Scan parquet (122) + +- Exchange (130) + +- Filter (129) + +- Scan parquet (128) (1) Scan parquet @@ -126,524 +136,564 @@ Input [2]: [s_suppkey#X, s_nationkey#X] Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(23) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [o_orderkey#X, o_custkey#X] Arguments: (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: X + +(40) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [c_custkey#X, c_nationkey#X] Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(69) ReusedExchange [Reuses operator id: 58] +(78) ReusedExchange [Reuses operator id: 66] Output [2]: [n_nationkey#X, n_name#X] -(70) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(71) InputAdapter +(80) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(72) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(73) ShuffledHashJoinExecTransformer +(82) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(74) ProjectExecTransformer +(83) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(75) FlushableHashAggregateExecTransformer +(84) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) ProjectExecTransformer +(85) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) WholeStageCodegenTransformer (X) +(86) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(78) ColumnarExchange +(87) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(88) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(80) InputAdapter +(89) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(81) InputIteratorTransformer +(90) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(82) RegularHashAggregateExecTransformer +(91) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(83) WholeStageCodegenTransformer (X) +(92) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(84) ColumnarExchange +(93) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(94) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(95) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(86) InputAdapter +(96) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(87) InputIteratorTransformer +(97) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(88) SortExecTransformer +(98) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(89) WholeStageCodegenTransformer (X) +(99) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(90) VeloxColumnarToRowExec +(100) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(91) Scan parquet +(101) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(102) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(103) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(104) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(105) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Exchange +(106) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(98) Project +(108) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(102) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(104) Project +(114) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(105) Exchange +(115) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(116) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(117) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(118) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(110) Project +(120) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(111) Exchange +(121) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(122) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(113) Filter +(123) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(114) Exchange +(124) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(116) Project +(126) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(117) Exchange +(127) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(128) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(119) Filter +(129) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(120) Exchange +(130) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(122) Project +(132) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(123) HashAggregate +(133) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(124) Exchange +(134) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(135) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(126) Exchange +(136) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(137) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(128) AdaptiveSparkPlan +(138) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt index e2a1907549f0..de63f62ffc7e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt @@ -1,150 +1,165 @@ == Physical Plan == -AdaptiveSparkPlan (177) +AdaptiveSparkPlan (192) +- == Final Plan == - VeloxColumnarToRowExec (125) - +- ^ SortExecTransformer (123) - +- ^ InputIteratorTransformer (122) - +- ShuffleQueryStage (120) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ShuffleQueryStage (113) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ShuffleQueryStage (96) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ShuffleQueryStage (81) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ShuffleQueryStage (66) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ShuffleQueryStage (51) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ShuffleQueryStage (36) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ShuffleQueryStage (21) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ShuffleQueryStage (6) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ShuffleQueryStage (14) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ShuffleQueryStage (29) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ FilterExecTransformer (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ShuffleQueryStage (44) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ FilterExecTransformer (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ShuffleQueryStage (59) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ FilterExecTransformer (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ShuffleQueryStage (74) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ FilterExecTransformer (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ FilterExecTransformer (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ShuffleQueryStage (104) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ FilterExecTransformer (100) - +- ^ Scan parquet (99) + VeloxColumnarToRowExec (140) + +- ^ SortExecTransformer (138) + +- ^ InputIteratorTransformer (137) + +- ShuffleQueryStage (135) + +- ColumnarExchange (134) + +- VeloxAppendBatches (133) + +- ^ ProjectExecTransformer (131) + +- ^ RegularHashAggregateExecTransformer (130) + +- ^ InputIteratorTransformer (129) + +- ShuffleQueryStage (127) + +- ColumnarExchange (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ FilterExecTransformer (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ FilterExecTransformer (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ FilterExecTransformer (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ FilterExecTransformer (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ FilterExecTransformer (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ FilterExecTransformer (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ FilterExecTransformer (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (176) - +- Exchange (175) - +- HashAggregate (174) - +- Exchange (173) - +- HashAggregate (172) - +- Project (171) - +- ShuffledHashJoin Inner BuildRight (170) - :- Exchange (165) - : +- Project (164) - : +- ShuffledHashJoin Inner BuildRight (163) - : :- Exchange (159) - : : +- Project (158) - : : +- ShuffledHashJoin Inner BuildRight (157) - : : :- Exchange (153) - : : : +- Project (152) - : : : +- ShuffledHashJoin Inner BuildRight (151) - : : : :- Exchange (147) - : : : : +- Project (146) - : : : : +- ShuffledHashJoin Inner BuildRight (145) - : : : : :- Exchange (141) - : : : : : +- Project (140) - : : : : : +- ShuffledHashJoin Inner BuildRight (139) - : : : : : :- Exchange (135) - : : : : : : +- Project (134) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) - : : : : : : :- Exchange (129) - : : : : : : : +- Project (128) - : : : : : : : +- Filter (127) - : : : : : : : +- Scan parquet (126) - : : : : : : +- Exchange (132) - : : : : : : +- Filter (131) - : : : : : : +- Scan parquet (130) - : : : : : +- Exchange (138) - : : : : : +- Filter (137) - : : : : : +- Scan parquet (136) - : : : : +- Exchange (144) - : : : : +- Filter (143) - : : : : +- Scan parquet (142) - : : : +- Exchange (150) - : : : +- Filter (149) - : : : +- Scan parquet (148) - : : +- Exchange (156) - : : +- Filter (155) - : : +- Scan parquet (154) - : +- Exchange (162) - : +- Filter (161) - : +- Scan parquet (160) - +- Exchange (169) - +- Project (168) - +- Filter (167) - +- Scan parquet (166) + Sort (191) + +- Exchange (190) + +- HashAggregate (189) + +- Exchange (188) + +- HashAggregate (187) + +- Project (186) + +- ShuffledHashJoin Inner BuildRight (185) + :- Exchange (180) + : +- Project (179) + : +- ShuffledHashJoin Inner BuildRight (178) + : :- Exchange (174) + : : +- Project (173) + : : +- ShuffledHashJoin Inner BuildRight (172) + : : :- Exchange (168) + : : : +- Project (167) + : : : +- ShuffledHashJoin Inner BuildRight (166) + : : : :- Exchange (162) + : : : : +- Project (161) + : : : : +- ShuffledHashJoin Inner BuildRight (160) + : : : : :- Exchange (156) + : : : : : +- Project (155) + : : : : : +- ShuffledHashJoin Inner BuildRight (154) + : : : : : :- Exchange (150) + : : : : : : +- Project (149) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (148) + : : : : : : :- Exchange (144) + : : : : : : : +- Project (143) + : : : : : : : +- Filter (142) + : : : : : : : +- Scan parquet (141) + : : : : : : +- Exchange (147) + : : : : : : +- Filter (146) + : : : : : : +- Scan parquet (145) + : : : : : +- Exchange (153) + : : : : : +- Filter (152) + : : : : : +- Scan parquet (151) + : : : : +- Exchange (159) + : : : : +- Filter (158) + : : : : +- Scan parquet (157) + : : : +- Exchange (165) + : : : +- Filter (164) + : : : +- Scan parquet (163) + : : +- Exchange (171) + : : +- Filter (170) + : : +- Scan parquet (169) + : +- Exchange (177) + : +- Filter (176) + : +- Scan parquet (175) + +- Exchange (184) + +- Project (183) + +- Filter (182) + +- Scan parquet (181) (1) Scan parquet @@ -166,732 +181,792 @@ Input [2]: [p_partkey#X, p_type#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [c_custkey#X, c_nationkey#X] Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(79) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(80) ColumnarExchange +(90) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: X + +(91) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(92) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(82) InputAdapter +(93) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(83) InputIteratorTransformer +(94) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(84) Scan parquet +(95) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) FilterExecTransformer +(96) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(86) ProjectExecTransformer +(97) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(92) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(93) ProjectExecTransformer +(105) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(94) WholeStageCodegenTransformer (X) +(106) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(95) ColumnarExchange +(107) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: X + +(108) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(109) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(97) InputAdapter +(110) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(98) InputIteratorTransformer +(111) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(99) Scan parquet +(112) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(100) FilterExecTransformer +(113) FilterExecTransformer Input [2]: [r_regionkey#X, r_name#X] Arguments: ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(101) ProjectExecTransformer +(114) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(102) WholeStageCodegenTransformer (X) +(115) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(103) ColumnarExchange +(116) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(117) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(104) ShuffleQueryStage +(118) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(105) InputAdapter +(119) InputAdapter Input [1]: [r_regionkey#X] -(106) InputIteratorTransformer +(120) InputIteratorTransformer Input [1]: [r_regionkey#X] -(107) ShuffledHashJoinExecTransformer +(121) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(108) ProjectExecTransformer +(122) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(109) FlushableHashAggregateExecTransformer +(123) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(110) ProjectExecTransformer +(124) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(111) WholeStageCodegenTransformer (X) +(125) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(112) ColumnarExchange +(126) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(113) ShuffleQueryStage +(127) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(114) InputAdapter +(128) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(115) InputIteratorTransformer +(129) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(116) RegularHashAggregateExecTransformer +(130) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(117) ProjectExecTransformer +(131) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(118) WholeStageCodegenTransformer (X) +(132) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(119) ColumnarExchange +(133) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(134) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(120) ShuffleQueryStage +(135) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(121) InputAdapter +(136) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(122) InputIteratorTransformer +(137) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(123) SortExecTransformer +(138) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(124) WholeStageCodegenTransformer (X) +(139) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(125) VeloxColumnarToRowExec +(140) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(126) Scan parquet +(141) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(127) Filter +(142) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(128) Project +(143) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(129) Exchange +(144) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) Scan parquet +(145) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(131) Filter +(146) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(132) Exchange +(147) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) ShuffledHashJoin +(148) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(134) Project +(149) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(135) Exchange +(150) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Scan parquet +(151) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(137) Filter +(152) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(138) Exchange +(153) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(154) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(140) Project +(155) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(141) Exchange +(156) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) Scan parquet +(157) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(143) Filter +(158) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(144) Exchange +(159) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) ShuffledHashJoin +(160) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(146) Project +(161) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(147) Exchange +(162) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Scan parquet +(163) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(149) Filter +(164) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(150) Exchange +(165) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) ShuffledHashJoin +(166) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(152) Project +(167) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(153) Exchange +(168) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) Scan parquet +(169) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(155) Filter +(170) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(156) Exchange +(171) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) ShuffledHashJoin +(172) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(158) Project +(173) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(159) Exchange +(174) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Scan parquet +(175) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(161) Filter +(176) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(162) Exchange +(177) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) ShuffledHashJoin +(178) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(164) Project +(179) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(165) Exchange +(180) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) Scan parquet +(181) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(167) Filter +(182) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(168) Project +(183) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(169) Exchange +(184) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) ShuffledHashJoin +(185) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(171) Project +(186) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(172) HashAggregate +(187) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(173) Exchange +(188) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(174) HashAggregate +(189) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(175) Exchange +(190) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(191) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(177) AdaptiveSparkPlan +(192) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt index b86e86d560e5..1fc81ef08365 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt @@ -1,114 +1,125 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (144) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- HashAggregate (130) - +- Exchange (129) - +- HashAggregate (128) - +- Project (127) - +- ShuffledHashJoin Inner BuildRight (126) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (98) - : : : : : +- Project (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (125) - +- Filter (124) - +- Scan parquet (123) + Sort (143) + +- Exchange (142) + +- HashAggregate (141) + +- Exchange (140) + +- HashAggregate (139) + +- Project (138) + +- ShuffledHashJoin Inner BuildRight (137) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (109) + : : : : : +- Project (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -130,548 +141,592 @@ Input [2]: [p_partkey#X, p_name#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: X + +(57) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: isnotnull(o_orderkey#X) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: X + +(74) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(99) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(97) Project +(108) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(109) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(110) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(100) Filter +(111) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(101) Exchange +(112) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(103) Project +(114) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(115) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(118) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(109) Project +(120) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(121) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(113) Exchange +(124) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(115) Project +(126) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(116) Exchange +(127) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(119) Exchange +(130) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(121) Project +(132) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(122) Exchange +(133) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(125) Exchange +(136) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(137) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(127) Project +(138) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(128) HashAggregate +(139) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(129) Exchange +(140) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) HashAggregate +(141) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(131) Exchange +(142) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(143) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(133) AdaptiveSparkPlan +(144) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/1.txt index 89de3133895b..2f3257bbdafc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt index 680971de4c1d..8ca3c5d087a8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt @@ -1,78 +1,84 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- TakeOrderedAndProjectExecTransformer (59) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ShuffleQueryStage (36), Statistics(X) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ShuffleQueryStage (21), Statistics(X) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ShuffleQueryStage (29), Statistics(X) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ FilterExecTransformer (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (66) + +- TakeOrderedAndProjectExecTransformer (65) + +- ^ ProjectExecTransformer (63) + +- ^ RegularHashAggregateExecTransformer (62) + +- ^ InputIteratorTransformer (61) + +- ShuffleQueryStage (59), Statistics(X) + +- ColumnarExchange (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24), Statistics(X) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33), Statistics(X) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FilterExecTransformer (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- ShuffledHashJoin Inner BuildRight (81) - :- Exchange (77) - : +- Project (76) - : +- ShuffledHashJoin Inner BuildRight (75) - : :- Exchange (70) - : : +- Project (69) - : : +- ShuffledHashJoin Inner BuildRight (68) - : : :- Exchange (63) - : : : +- Filter (62) - : : : +- Scan parquet (61) - : : +- Exchange (67) - : : +- Project (66) - : : +- Filter (65) - : : +- Scan parquet (64) - : +- Exchange (74) - : +- Project (73) - : +- Filter (72) - : +- Scan parquet (71) - +- Exchange (80) - +- Filter (79) - +- Scan parquet (78) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- ShuffledHashJoin Inner BuildRight (87) + :- Exchange (83) + : +- Project (82) + : +- ShuffledHashJoin Inner BuildRight (81) + : :- Exchange (76) + : : +- Project (75) + : : +- ShuffledHashJoin Inner BuildRight (74) + : : :- Exchange (69) + : : : +- Filter (68) + : : : +- Scan parquet (67) + : : +- Exchange (73) + : : +- Project (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (80) + : +- Project (79) + : +- Filter (78) + : +- Scan parquet (77) + +- Exchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -94,358 +100,382 @@ Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acct Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: X + +(6) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Arguments: ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(55) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(56) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(58) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(59) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(60) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(61) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(62) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(57) ProjectExecTransformer +(63) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(58) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(59) TakeOrderedAndProjectExecTransformer +(65) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(60) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(61) Scan parquet +(67) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) Filter +(68) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(63) Exchange +(69) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(65) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(66) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(67) Exchange +(73) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) ShuffledHashJoin +(74) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(69) Project +(75) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(70) Exchange +(76) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(77) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(72) Filter +(78) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(73) Project +(79) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(74) Exchange +(80) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(81) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(76) Project +(82) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(77) Exchange +(83) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(80) Exchange +(86) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(87) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(88) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(89) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(84) Exchange +(90) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(91) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(86) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt index b964ec1ed8d4..0e370c4f114a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt @@ -1,65 +1,70 @@ == Physical Plan == -AdaptiveSparkPlan (72) +AdaptiveSparkPlan (77) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ SortExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ShuffleQueryStage (38), Statistics(X) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (55) + +- ^ SortExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ FilterExecTransformer (46) + +- ^ RegularHashAggregateExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42), Statistics(X) + +- ColumnarExchange (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (71) - +- Exchange (70) - +- Filter (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- ShuffledHashJoin Inner BuildRight (64) - :- Exchange (59) - : +- Project (58) - : +- ShuffledHashJoin Inner BuildRight (57) - : :- Exchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- Exchange (56) - : +- Filter (55) - : +- Scan parquet (54) - +- Exchange (63) - +- Project (62) - +- Filter (61) - +- Scan parquet (60) + Sort (76) + +- Exchange (75) + +- Filter (74) + +- HashAggregate (73) + +- Exchange (72) + +- HashAggregate (71) + +- Project (70) + +- ShuffledHashJoin Inner BuildRight (69) + :- Exchange (64) + : +- Project (63) + : +- ShuffledHashJoin Inner BuildRight (62) + : :- Exchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- Exchange (61) + : +- Filter (60) + : +- Scan parquet (59) + +- Exchange (68) + +- Project (67) + +- Filter (66) + +- Scan parquet (65) (1) Scan parquet @@ -81,549 +86,579 @@ Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [1]: [n_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [n_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(34) FlushableHashAggregateExecTransformer +(38) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(41) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(43) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(45) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(42) FilterExecTransformer +(46) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(43) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(44) ColumnarExchange +(48) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(49) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(46) InputAdapter +(51) InputAdapter Input [2]: [ps_partkey#X, value#X] -(47) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(48) SortExecTransformer +(53) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(49) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(50) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(51) Scan parquet +(56) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(52) Filter +(57) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(53) Exchange +(58) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(59) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(60) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) Exchange +(61) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(62) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(58) Project +(63) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(59) Exchange +(64) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(65) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(61) Filter +(66) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(62) Project +(67) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(63) Exchange +(68) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(69) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(65) Project +(70) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(66) HashAggregate +(71) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(67) Exchange +(72) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(73) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(69) Filter +(74) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(70) Exchange +(75) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(76) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(72) AdaptiveSparkPlan +(77) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (120) +Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (127) +- == Final Plan == - VeloxColumnarToRowExec (102) - +- ^ ProjectExecTransformer (100) - +- ^ RegularHashAggregateExecTransformer (99) - +- ^ RegularHashAggregateExecTransformer (98) - +- ^ ProjectExecTransformer (97) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) - :- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) - : :- ^ InputIteratorTransformer (80) - : : +- ShuffleQueryStage (78), Statistics(X) - : : +- ColumnarExchange (77) - : : +- ^ ProjectExecTransformer (75) - : : +- ^ FilterExecTransformer (74) - : : +- ^ Scan parquet (73) - : +- ^ InputIteratorTransformer (84) - : +- ShuffleQueryStage (82), Statistics(X) - : +- ReusedExchange (81) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ReusedExchange (92) + VeloxColumnarToRowExec (109) + +- ^ ProjectExecTransformer (107) + +- ^ RegularHashAggregateExecTransformer (106) + +- ^ RegularHashAggregateExecTransformer (105) + +- ^ ProjectExecTransformer (104) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (103) + :- ^ InputIteratorTransformer (98) + : +- ShuffleQueryStage (96), Statistics(X) + : +- ColumnarExchange (95) + : +- VeloxAppendBatches (94) + : +- ^ ProjectExecTransformer (92) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (91) + : :- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ FilterExecTransformer (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (90) + : +- ShuffleQueryStage (88), Statistics(X) + : +- ReusedExchange (87) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ReusedExchange (99) +- == Initial Plan == - HashAggregate (119) - +- HashAggregate (118) - +- Project (117) - +- ShuffledHashJoin Inner BuildRight (116) - :- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildRight (109) - : :- Exchange (105) - : : +- Filter (104) - : : +- Scan parquet (103) - : +- Exchange (108) - : +- Filter (107) - : +- Scan parquet (106) - +- Exchange (115) - +- Project (114) - +- Filter (113) - +- Scan parquet (112) - - -(73) Scan parquet + HashAggregate (126) + +- HashAggregate (125) + +- Project (124) + +- ShuffledHashJoin Inner BuildRight (123) + :- Exchange (118) + : +- Project (117) + : +- ShuffledHashJoin Inner BuildRight (116) + : :- Exchange (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- Exchange (115) + : +- Filter (114) + : +- Scan parquet (113) + +- Exchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) + + +(78) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(74) FilterExecTransformer +(79) FilterExecTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: isnotnull(ps_suppkey#X) -(75) ProjectExecTransformer +(80) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(76) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(77) ColumnarExchange +(82) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(83) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(78) ShuffleQueryStage +(84) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(79) InputAdapter +(85) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(80) InputIteratorTransformer +(86) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(81) ReusedExchange [Reuses operator id: 13] +(87) ReusedExchange [Reuses operator id: 15] Output [2]: [s_suppkey#X, s_nationkey#X] -(82) ShuffleQueryStage +(88) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(83) InputAdapter +(89) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(84) InputIteratorTransformer +(90) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(85) ShuffledHashJoinExecTransformer +(91) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(86) ProjectExecTransformer +(92) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(87) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(88) ColumnarExchange +(94) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(95) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(96) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(90) InputAdapter +(97) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(91) InputIteratorTransformer +(98) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(92) ReusedExchange [Reuses operator id: 28] +(99) ReusedExchange [Reuses operator id: 32] Output [1]: [n_nationkey#X] -(93) ShuffleQueryStage +(100) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(94) InputAdapter +(101) InputAdapter Input [1]: [n_nationkey#X] -(95) InputIteratorTransformer +(102) InputIteratorTransformer Input [1]: [n_nationkey#X] -(96) ShuffledHashJoinExecTransformer +(103) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(97) ProjectExecTransformer +(104) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(98) RegularHashAggregateExecTransformer +(105) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(99) RegularHashAggregateExecTransformer +(106) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(100) ProjectExecTransformer +(107) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(101) WholeStageCodegenTransformer (X) +(108) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(102) VeloxColumnarToRowExec +(109) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(103) Scan parquet +(110) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(104) Filter +(111) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(105) Exchange +(112) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(113) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(107) Filter +(114) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(108) Exchange +(115) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(116) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(110) Project +(117) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(111) Exchange +(118) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(119) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(113) Filter +(120) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(114) Project +(121) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(115) Exchange +(122) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(123) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(117) Project +(124) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(118) HashAggregate +(125) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(119) HashAggregate +(126) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(120) AdaptiveSparkPlan +(127) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt index fabde3ecd687..0ee1a20a6c39 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt @@ -1,46 +1,49 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin Inner BuildLeft (42) - :- Exchange (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- ShuffledHashJoin Inner BuildLeft (45) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -62,196 +65,208 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Arguments: ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(31) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) Scan parquet +(38) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(36) Filter +(39) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(37) Exchange +(40) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(41) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(40) Project +(43) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(41) Exchange +(44) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(45) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(43) Project +(46) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(44) HashAggregate +(47) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(45) Exchange +(48) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(49) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(47) Exchange +(50) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(51) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt index 6ca6e75ac545..be5442000bde 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt @@ -1,49 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftOuter BuildRight (43) - :- Exchange (38) - : +- Scan parquet (37) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftOuter BuildRight (46) + :- Exchange (41) + : +- Scan parquet (40) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -60,223 +63,235 @@ Input [1]: [c_custkey#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(4) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) FilterExecTransformer +(10) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Arguments: ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(44) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) Exchange +(45) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(47) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(48) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt index c1a4a2fe884f..afac58cb52bc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (23) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (25) + +- ^ ProjectExecTransformer (23) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (34) - +- HashAggregate (33) - +- Project (32) - +- ShuffledHashJoin Inner BuildRight (31) - :- Exchange (27) - : +- Project (26) - : +- Filter (25) - : +- Scan parquet (24) - +- Exchange (30) - +- Filter (29) - +- Scan parquet (28) + HashAggregate (36) + +- HashAggregate (35) + +- Project (34) + +- ShuffledHashJoin Inner BuildRight (33) + :- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -52,144 +54,152 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [2]: [p_partkey#X, p_type#X] Arguments: isnotnull(p_partkey#X) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(21) ProjectExecTransformer +(23) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(22) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(23) VeloxColumnarToRowExec +(25) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(24) Scan parquet +(26) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(26) Project +(28) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) Exchange +(29) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Scan parquet +(30) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(30) Exchange +(32) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) ShuffledHashJoin +(33) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(32) Project +(34) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(34) HashAggregate +(36) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] -(35) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt index a45c61431782..7b33a6b69bd9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt @@ -1,43 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- AQEShuffleRead (26) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (29) + +- AQEShuffleRead (28) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (22) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- Project (39) - +- ShuffledHashJoin Inner BuildLeft (38) - :- Exchange (30) - : +- Filter (29) - : +- Scan parquet (28) - +- Filter (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (43) + +- Exchange (42) + +- Project (41) + +- ShuffledHashJoin Inner BuildLeft (40) + :- Exchange (32) + : +- Filter (31) + : +- Scan parquet (30) + +- Filter (39) + +- HashAggregate (38) + +- Exchange (37) + +- HashAggregate (36) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -59,328 +61,336 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(16) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(20) FilterExecTransformer +(21) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(21) ShuffledHashJoinExecTransformer +(22) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(26) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) AQEShuffleRead +(28) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(27) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) Scan parquet +(30) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(30) Exchange +(32) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) Scan parquet +(33) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(32) Filter +(34) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(33) Project +(35) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(34) HashAggregate +(36) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(35) Exchange +(37) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(38) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(37) Filter +(39) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(38) ShuffledHashJoin +(40) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(39) Project +(41) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(40) Exchange +(42) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(43) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (67) +Subquery:1 Hosting operator id = 21 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ RegularHashAggregateExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ ProjectExecTransformer (47) - +- ^ FlushableHashAggregateExecTransformer (46) - +- ^ ProjectExecTransformer (45) - +- ^ FilterExecTransformer (44) - +- ^ Scan parquet (43) + VeloxColumnarToRowExec (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ ProjectExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ InputIteratorTransformer (54) + +- ShuffleQueryStage (52), Statistics(X) + +- ColumnarExchange (51) + +- ^ ProjectExecTransformer (49) + +- ^ FlushableHashAggregateExecTransformer (48) + +- ^ ProjectExecTransformer (47) + +- ^ FilterExecTransformer (46) + +- ^ Scan parquet (45) +- == Initial Plan == - HashAggregate (66) - +- HashAggregate (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (68) + +- HashAggregate (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- Filter (62) + +- Scan parquet (61) -(43) Scan parquet +(45) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(44) FilterExecTransformer +(46) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(45) ProjectExecTransformer +(47) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) FlushableHashAggregateExecTransformer +(48) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(48) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(49) ColumnarExchange +(51) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(51) InputAdapter +(53) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(53) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(54) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(55) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(56) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(57) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(58) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(59) Scan parquet +(61) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(60) Filter +(62) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(61) Project +(63) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(63) Exchange +(65) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(65) HashAggregate +(67) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(66) HashAggregate +(68) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(67) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt index 9a927b77805e..287ae9f00ed1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt @@ -1,59 +1,62 @@ == Physical Plan == -AdaptiveSparkPlan (64) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (42) - +- ^ SortExecTransformer (40) - +- ^ InputIteratorTransformer (39) - +- ShuffleQueryStage (37), Statistics(X) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (45) + +- ^ SortExecTransformer (43) + +- ^ InputIteratorTransformer (42) + +- ShuffleQueryStage (40), Statistics(X) + +- ColumnarExchange (39) + +- VeloxAppendBatches (38) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- ^ ProjectExecTransformer (30) + +- ^ FlushableHashAggregateExecTransformer (29) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (63) - +- Exchange (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- HashAggregate (58) - +- Exchange (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (50) - : +- BroadcastHashJoin LeftAnti BuildRight (49) - : :- Filter (44) - : : +- Scan parquet (43) - : +- BroadcastExchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (53) - +- Filter (52) - +- Scan parquet (51) + Sort (66) + +- Exchange (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- HashAggregate (61) + +- Exchange (60) + +- HashAggregate (59) + +- Project (58) + +- ShuffledHashJoin Inner BuildRight (57) + :- Exchange (53) + : +- BroadcastHashJoin LeftAnti BuildRight (52) + : :- Filter (47) + : : +- Scan parquet (46) + : +- BroadcastExchange (51) + : +- Project (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -75,270 +78,282 @@ Input [2]: [ps_partkey#X, ps_suppkey#X] Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) FlushableHashAggregateExecTransformer +(29) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(30) ColumnarExchange +(32) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(32) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) ColumnarExchange +(38) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(39) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(40) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(38) InputAdapter +(41) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) InputIteratorTransformer +(42) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(40) SortExecTransformer +(43) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(41) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(42) VeloxColumnarToRowExec +(45) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) Scan parquet +(46) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(45) Scan parquet +(48) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(46) Filter +(49) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(47) Project +(50) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(48) BroadcastExchange +(51) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(49) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(50) Exchange +(53) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(54) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(53) Exchange +(56) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(57) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(55) Project +(58) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(56) HashAggregate +(59) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) HashAggregate +(61) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(59) HashAggregate +(62) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(60) Exchange +(63) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(62) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) Sort +(66) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(64) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt index 48ba7ddab093..23d34ad9b587 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt @@ -1,56 +1,58 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ ProjectExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) - :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ FilterExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ FilterExecTransformer (20) - +- ^ Scan parquet (19) + VeloxColumnarToRowExec (39) + +- ^ ProjectExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ ProjectExecTransformer (34) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (33) + :- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ FilterExecTransformer (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (56) - +- HashAggregate (55) - +- Project (54) - +- ShuffledHashJoin Inner BuildRight (53) - :- Project (46) - : +- ShuffledHashJoin Inner BuildRight (45) - : :- Exchange (40) - : : +- Filter (39) - : : +- Scan parquet (38) - : +- Exchange (44) - : +- Project (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Filter (48) - +- Scan parquet (47) + HashAggregate (58) + +- HashAggregate (57) + +- Project (56) + +- ShuffledHashJoin Inner BuildRight (55) + :- Project (48) + : +- ShuffledHashJoin Inner BuildRight (47) + : :- Exchange (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Exchange (46) + : +- Project (45) + : +- Filter (44) + : +- Scan parquet (43) + +- Filter (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Filter (50) + +- Scan parquet (49) (1) Scan parquet @@ -72,250 +74,258 @@ Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [p_partkey#X, p_brand#X, p_container#X] Arguments: ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(19) Scan parquet +(21) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(20) FilterExecTransformer +(22) FilterExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Arguments: isnotnull(l_partkey#X) -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(30) FilterExecTransformer +(32) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(31) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(32) ProjectExecTransformer +(34) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(33) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(35) ProjectExecTransformer +(37) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(36) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(37) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(38) Scan parquet +(40) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(39) Filter +(41) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(40) Exchange +(42) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(43) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(43) Project +(45) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(44) Exchange +(46) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(46) Project +(48) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(47) Scan parquet +(49) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(48) Filter +(50) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(49) HashAggregate +(51) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(50) Exchange +(52) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) Filter +(54) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(53) ShuffledHashJoin +(55) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(54) Project +(56) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) HashAggregate +(57) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(56) HashAggregate +(58) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] -(57) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt index a15a185bbd77..fdf15018cafa 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt @@ -1,89 +1,94 @@ == Physical Plan == -AdaptiveSparkPlan (97) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- TakeOrderedAndProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) - :- ^ InputIteratorTransformer (41) - : +- ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ShuffleQueryStage (22), Statistics(X) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) - :- ^ InputIteratorTransformer (49) - : +- ShuffleQueryStage (47), Statistics(X) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ FilterExecTransformer (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ShuffleQueryStage (51), Statistics(X) - +- ReusedExchange (50) + VeloxColumnarToRowExec (69) + +- TakeOrderedAndProjectExecTransformer (68) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ RegularHashAggregateExecTransformer (65) + +- ^ ProjectExecTransformer (64) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (63) + :- ^ InputIteratorTransformer (45) + : +- ShuffleQueryStage (43), Statistics(X) + : +- ColumnarExchange (42) + : +- VeloxAppendBatches (41) + : +- ^ ProjectExecTransformer (39) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (38) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (37) + : +- ShuffleQueryStage (35), Statistics(X) + : +- ColumnarExchange (34) + : +- VeloxAppendBatches (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (30) + : :- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ Scan parquet (10) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ RegularHashAggregateExecTransformer (27) + : +- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) + :- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- VeloxAppendBatches (50) + : +- ^ ProjectExecTransformer (48) + : +- ^ FilterExecTransformer (47) + : +- ^ Scan parquet (46) + +- ^ ProjectExecTransformer (61) + +- ^ FilterExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ReusedExchange (55) +- == Initial Plan == - TakeOrderedAndProject (96) - +- HashAggregate (95) - +- HashAggregate (94) - +- Project (93) - +- ShuffledHashJoin Inner BuildRight (92) - :- Exchange (81) - : +- Project (80) - : +- ShuffledHashJoin Inner BuildLeft (79) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (78) - : +- ShuffledHashJoin LeftSemi BuildRight (77) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Project (76) - : +- Filter (75) - : +- HashAggregate (74) - : +- Exchange (73) - : +- HashAggregate (72) - : +- Scan parquet (71) - +- ShuffledHashJoin LeftSemi BuildRight (91) - :- Exchange (84) - : +- Filter (83) - : +- Scan parquet (82) - +- Project (90) - +- Filter (89) - +- HashAggregate (88) - +- Exchange (87) - +- HashAggregate (86) - +- Scan parquet (85) + TakeOrderedAndProject (101) + +- HashAggregate (100) + +- HashAggregate (99) + +- Project (98) + +- ShuffledHashJoin Inner BuildRight (97) + :- Exchange (86) + : +- Project (85) + : +- ShuffledHashJoin Inner BuildLeft (84) + : :- Exchange (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (83) + : +- ShuffledHashJoin LeftSemi BuildRight (82) + : :- Exchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- Project (81) + : +- Filter (80) + : +- HashAggregate (79) + : +- Exchange (78) + : +- HashAggregate (77) + : +- Scan parquet (76) + +- ShuffledHashJoin LeftSemi BuildRight (96) + :- Exchange (89) + : +- Filter (88) + : +- Scan parquet (87) + +- Project (95) + +- Filter (94) + +- HashAggregate (93) + +- Exchange (92) + +- HashAggregate (91) + +- Scan parquet (90) (1) Scan parquet @@ -105,420 +110,440 @@ Input [2]: [c_custkey#X, c_name#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(26) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(27) ProjectExecTransformer +(29) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(28) ShuffledHashJoinExecTransformer +(30) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(30) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(31) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(33) InputAdapter +(36) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(34) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) ShuffledHashJoinExecTransformer +(38) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(36) ProjectExecTransformer +(39) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(38) ColumnarExchange +(41) VeloxAppendBatches +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(42) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(43) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(40) InputAdapter +(44) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(41) InputIteratorTransformer +(45) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(42) Scan parquet +(46) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(43) FilterExecTransformer +(47) FilterExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Arguments: isnotnull(l_orderkey#X) -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(46) ColumnarExchange +(50) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: X + +(51) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(52) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(48) InputAdapter +(53) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(49) InputIteratorTransformer +(54) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(50) ReusedExchange [Reuses operator id: 21] +(55) ReusedExchange [Reuses operator id: 23] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) ShuffleQueryStage +(56) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(52) InputAdapter +(57) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(53) InputIteratorTransformer +(58) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(54) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(55) FilterExecTransformer +(60) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(56) ProjectExecTransformer +(61) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(58) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(59) ProjectExecTransformer +(64) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(60) RegularHashAggregateExecTransformer +(65) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(61) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(62) WholeStageCodegenTransformer (X) +(67) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(63) TakeOrderedAndProjectExecTransformer +(68) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(64) VeloxColumnarToRowExec +(69) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(65) Scan parquet +(70) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(66) Filter +(71) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(67) Exchange +(72) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(73) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(69) Filter +(74) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(70) Exchange +(75) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(76) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(77) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(78) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(79) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(80) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(81) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Exchange +(83) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) ShuffledHashJoin +(84) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(80) Project +(85) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(81) Exchange +(86) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(87) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(83) Filter +(88) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(84) Exchange +(89) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(90) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(86) HashAggregate +(91) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(87) Exchange +(92) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) HashAggregate +(93) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(89) Filter +(94) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(90) Project +(95) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(91) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(93) Project +(98) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(94) HashAggregate +(99) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(95) HashAggregate +(100) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(96) TakeOrderedAndProject +(101) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(97) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt index 80f0189d5d65..14a5515d1e79 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (33) - +- HashAggregate (32) - +- Project (31) - +- ShuffledHashJoin Inner BuildRight (30) - :- Exchange (26) - : +- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- Exchange (29) - +- Filter (28) - +- Scan parquet (27) + HashAggregate (35) + +- HashAggregate (34) + +- Project (33) + +- ShuffledHashJoin Inner BuildRight (32) + :- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) (1) Scan parquet @@ -51,140 +53,148 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(24) VeloxColumnarToRowExec Input [1]: [revenue#X] -(23) Scan parquet +(25) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(24) Filter +(26) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(25) Project +(27) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(26) Exchange +(28) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Scan parquet +(29) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(28) Filter +(30) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(29) Exchange +(31) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) ShuffledHashJoin +(32) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(31) Project +(33) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(32) HashAggregate +(34) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(34) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt index a0dcc7029a35..7b840720bc90 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt @@ -1,109 +1,119 @@ == Physical Plan == -AdaptiveSparkPlan (123) +AdaptiveSparkPlan (133) +- == Final Plan == - VeloxColumnarToRowExec (83) - +- AQEShuffleRead (82) - +- ShuffleQueryStage (81), Statistics(X) - +- ColumnarExchange (80) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ShuffleQueryStage (29), Statistics(X) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ShuffleQueryStage (14), Statistics(X) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ FilterExecTransformer (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ShuffleQueryStage (22), Statistics(X) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ FilterExecTransformer (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ FilterExecTransformer (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41), Statistics(X) - : +- ReusedExchange (40) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (93) + +- AQEShuffleRead (92) + +- ShuffleQueryStage (91), Statistics(X) + +- ColumnarExchange (90) + +- VeloxAppendBatches (89) + +- ^ ProjectExecTransformer (87) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (86) + :- ^ InputIteratorTransformer (76) + : +- ShuffleQueryStage (74), Statistics(X) + : +- ColumnarExchange (73) + : +- VeloxAppendBatches (72) + : +- ^ ProjectExecTransformer (70) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (69) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- VeloxAppendBatches (64) + : +- ^ ProjectExecTransformer (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (61) + : :- ^ InputIteratorTransformer (35) + : : +- ShuffleQueryStage (33), Statistics(X) + : : +- ColumnarExchange (32) + : : +- VeloxAppendBatches (31) + : : +- ^ ProjectExecTransformer (29) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : : :- ^ InputIteratorTransformer (18) + : : : +- ShuffleQueryStage (16), Statistics(X) + : : : +- ColumnarExchange (15) + : : : +- VeloxAppendBatches (14) + : : : +- ^ ProjectExecTransformer (12) + : : : +- ^ FilterExecTransformer (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (27) + : : +- ShuffleQueryStage (25), Statistics(X) + : : +- ColumnarExchange (24) + : : +- VeloxAppendBatches (23) + : : +- ^ ProjectExecTransformer (21) + : : +- ^ FilterExecTransformer (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (60) + : +- ShuffleQueryStage (58), Statistics(X) + : +- ColumnarExchange (57) + : +- VeloxAppendBatches (56) + : +- ^ ProjectExecTransformer (54) + : +- ^ FilterExecTransformer (53) + : +- ^ ProjectExecTransformer (52) + : +- ^ RegularHashAggregateExecTransformer (51) + : +- ^ RegularHashAggregateExecTransformer (50) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (49) + : :- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ FilterExecTransformer (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (48) + : +- ShuffleQueryStage (46), Statistics(X) + : +- ReusedExchange (45) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- VeloxAppendBatches (81) + +- ^ ProjectExecTransformer (79) + +- ^ FilterExecTransformer (78) + +- ^ Scan parquet (77) +- == Initial Plan == - Sort (122) - +- Exchange (121) - +- Project (120) - +- ShuffledHashJoin Inner BuildRight (119) - :- Exchange (114) - : +- Project (113) - : +- ShuffledHashJoin LeftSemi BuildRight (112) - : :- Exchange (86) - : : +- Filter (85) - : : +- Scan parquet (84) - : +- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildLeft (109) - : :- Exchange (95) - : : +- ShuffledHashJoin LeftSemi BuildRight (94) - : : :- Exchange (89) - : : : +- Filter (88) - : : : +- Scan parquet (87) - : : +- Exchange (93) - : : +- Project (92) - : : +- Filter (91) - : : +- Scan parquet (90) - : +- Exchange (108) - : +- Filter (107) - : +- HashAggregate (106) - : +- HashAggregate (105) - : +- ShuffledHashJoin LeftSemi BuildRight (104) - : :- Exchange (99) - : : +- Project (98) - : : +- Filter (97) - : : +- Scan parquet (96) - : +- Exchange (103) - : +- Project (102) - : +- Filter (101) - : +- Scan parquet (100) - +- Exchange (118) - +- Project (117) - +- Filter (116) - +- Scan parquet (115) + Sort (132) + +- Exchange (131) + +- Project (130) + +- ShuffledHashJoin Inner BuildRight (129) + :- Exchange (124) + : +- Project (123) + : +- ShuffledHashJoin LeftSemi BuildRight (122) + : :- Exchange (96) + : : +- Filter (95) + : : +- Scan parquet (94) + : +- Exchange (121) + : +- Project (120) + : +- ShuffledHashJoin Inner BuildLeft (119) + : :- Exchange (105) + : : +- ShuffledHashJoin LeftSemi BuildRight (104) + : : :- Exchange (99) + : : : +- Filter (98) + : : : +- Scan parquet (97) + : : +- Exchange (103) + : : +- Project (102) + : : +- Filter (101) + : : +- Scan parquet (100) + : +- Exchange (118) + : +- Filter (117) + : +- HashAggregate (116) + : +- HashAggregate (115) + : +- ShuffledHashJoin LeftSemi BuildRight (114) + : :- Exchange (109) + : : +- Project (108) + : : +- Filter (107) + : : +- Scan parquet (106) + : +- Exchange (113) + : +- Project (112) + : +- Filter (111) + : +- Scan parquet (110) + +- Exchange (128) + +- Project (127) + +- Filter (126) + +- Scan parquet (125) (1) Scan parquet @@ -125,508 +135,548 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(18) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [p_partkey#X, p_name#X] Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [1]: [p_partkey#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [1]: [p_partkey#X] -(25) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(32) Scan parquet +(36) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(33) FilterExecTransformer +(37) FilterExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(40) ReusedExchange [Reuses operator id: 21] +(45) ReusedExchange [Reuses operator id: 24] Output [1]: [p_partkey#X] -(41) ShuffleQueryStage +(46) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(42) InputAdapter +(47) InputAdapter Input [1]: [p_partkey#X] -(43) InputIteratorTransformer +(48) InputIteratorTransformer Input [1]: [p_partkey#X] -(44) ShuffledHashJoinExecTransformer +(49) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(45) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(46) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(47) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(48) FilterExecTransformer +(53) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(49) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(51) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(53) InputAdapter +(59) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(54) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(55) ShuffledHashJoinExecTransformer +(61) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(58) ColumnarExchange +(64) VeloxAppendBatches +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: X + +(65) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(66) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(60) InputAdapter +(67) InputAdapter Input [1]: [ps_suppkey#X] -(61) InputIteratorTransformer +(68) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(62) ShuffledHashJoinExecTransformer +(69) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(63) ProjectExecTransformer +(70) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(71) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(72) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(73) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(74) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(75) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(68) InputIteratorTransformer +(76) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(69) Scan parquet +(77) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer +(78) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(71) ProjectExecTransformer +(79) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(73) ColumnarExchange +(81) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(82) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(83) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(75) InputAdapter +(84) InputAdapter Input [1]: [n_nationkey#X] -(76) InputIteratorTransformer +(85) InputIteratorTransformer Input [1]: [n_nationkey#X] -(77) ShuffledHashJoinExecTransformer +(86) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(87) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(79) WholeStageCodegenTransformer (X) +(88) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(80) ColumnarExchange +(89) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(90) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(91) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(82) AQEShuffleRead +(92) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(83) VeloxColumnarToRowExec +(93) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(84) Scan parquet +(94) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(85) Filter +(95) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(86) Exchange +(96) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Scan parquet +(97) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(88) Filter +(98) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(89) Exchange +(99) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(100) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(91) Filter +(101) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(92) Project +(102) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(93) Exchange +(103) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) ShuffledHashJoin +(104) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(95) Exchange +(105) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) Scan parquet +(106) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(97) Filter +(107) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(98) Project +(108) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(102) Project +(112) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(103) Exchange +(113) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(105) HashAggregate +(115) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(106) HashAggregate +(116) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(107) Filter +(117) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(108) Exchange +(118) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(110) Project +(120) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(111) Exchange +(121) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(113) Project +(123) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(114) Exchange +(124) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) Scan parquet +(125) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(116) Filter +(126) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(117) Project +(127) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(118) Exchange +(128) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(129) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(120) Project +(130) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(121) Exchange +(131) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Sort +(132) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(123) AdaptiveSparkPlan +(133) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt index 3635363cfe47..245faea0cde8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt @@ -1,103 +1,112 @@ == Physical Plan == -AdaptiveSparkPlan (118) +AdaptiveSparkPlan (127) +- == Final Plan == - VeloxColumnarToRowExec (81) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ShuffleQueryStage (21), Statistics(X) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ShuffleQueryStage (30), Statistics(X) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ FilterExecTransformer (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ShuffleQueryStage (67), Statistics(X) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ FilterExecTransformer (63) - +- ^ Scan parquet (62) + VeloxColumnarToRowExec (90) + +- ^ RegularHashAggregateExecTransformer (88) + +- ^ InputIteratorTransformer (87) + +- ShuffleQueryStage (85), Statistics(X) + +- ColumnarExchange (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24), Statistics(X) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34), Statistics(X) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ FilterExecTransformer (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ FilterExecTransformer (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ FilterExecTransformer (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (117) - +- HashAggregate (116) - +- Exchange (115) - +- HashAggregate (114) - +- Project (113) - +- ShuffledHashJoin Inner BuildRight (112) - :- Exchange (107) - : +- Project (106) - : +- ShuffledHashJoin Inner BuildRight (105) - : :- Exchange (100) - : : +- Project (99) - : : +- ShuffledHashJoin Inner BuildLeft (98) - : : :- Exchange (84) - : : : +- Filter (83) - : : : +- Scan parquet (82) - : : +- Exchange (97) - : : +- ShuffledHashJoin LeftAnti BuildRight (96) - : : :- ShuffledHashJoin LeftSemi BuildRight (91) - : : : :- Exchange (88) - : : : : +- Project (87) - : : : : +- Filter (86) - : : : : +- Scan parquet (85) - : : : +- Exchange (90) - : : : +- Scan parquet (89) - : : +- Exchange (95) - : : +- Project (94) - : : +- Filter (93) - : : +- Scan parquet (92) - : +- Exchange (104) - : +- Project (103) - : +- Filter (102) - : +- Scan parquet (101) - +- Exchange (111) - +- Project (110) - +- Filter (109) - +- Scan parquet (108) + TakeOrderedAndProject (126) + +- HashAggregate (125) + +- Exchange (124) + +- HashAggregate (123) + +- Project (122) + +- ShuffledHashJoin Inner BuildRight (121) + :- Exchange (116) + : +- Project (115) + : +- ShuffledHashJoin Inner BuildRight (114) + : :- Exchange (109) + : : +- Project (108) + : : +- ShuffledHashJoin Inner BuildLeft (107) + : : :- Exchange (93) + : : : +- Filter (92) + : : : +- Scan parquet (91) + : : +- Exchange (106) + : : +- ShuffledHashJoin LeftAnti BuildRight (105) + : : :- ShuffledHashJoin LeftSemi BuildRight (100) + : : : :- Exchange (97) + : : : : +- Project (96) + : : : : +- Filter (95) + : : : : +- Scan parquet (94) + : : : +- Exchange (99) + : : : +- Scan parquet (98) + : : +- Exchange (104) + : : +- Project (103) + : : +- Filter (102) + : : +- Scan parquet (101) + : +- Exchange (113) + : +- Project (112) + : +- Filter (111) + : +- Scan parquet (110) + +- Exchange (120) + +- Project (119) + +- Filter (118) + +- Scan parquet (117) (1) Scan parquet @@ -119,490 +128,526 @@ Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(24) ShuffledHashJoinExecTransformer +(27) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(25) Scan parquet +(28) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(26) FilterExecTransformer +(29) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(29) ColumnarExchange +(32) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(33) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(33) ShuffledHashJoinExecTransformer +(37) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(41) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(40) ShuffledHashJoinExecTransformer +(45) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(47) Scan parquet +(53) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(48) FilterExecTransformer +(54) FilterExecTransformer Input [2]: [o_orderkey#X, o_orderstatus#X] Arguments: ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(49) ProjectExecTransformer +(55) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(50) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(51) ColumnarExchange +(57) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: X + +(58) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(53) InputAdapter +(60) InputAdapter Input [1]: [o_orderkey#X] -(54) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [o_orderkey#X] -(55) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(62) Scan parquet +(70) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(63) FilterExecTransformer +(71) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(64) ProjectExecTransformer +(72) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(65) WholeStageCodegenTransformer (X) +(73) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(66) ColumnarExchange +(74) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(75) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(76) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(68) InputAdapter +(77) InputAdapter Input [1]: [n_nationkey#X] -(69) InputIteratorTransformer +(78) InputIteratorTransformer Input [1]: [n_nationkey#X] -(70) ShuffledHashJoinExecTransformer +(79) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(72) FlushableHashAggregateExecTransformer +(81) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(73) ProjectExecTransformer +(82) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(74) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(75) ColumnarExchange +(84) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(85) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(77) InputAdapter +(86) InputAdapter Input [2]: [s_name#X, count#X] -(78) InputIteratorTransformer +(87) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(79) RegularHashAggregateExecTransformer +(88) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(80) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(81) VeloxColumnarToRowExec +(90) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(82) Scan parquet +(91) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(83) Filter +(92) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(84) Exchange +(93) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(94) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(86) Filter +(95) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(87) Project +(96) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(88) Exchange +(97) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) Scan parquet +(98) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(90) Exchange +(99) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) ShuffledHashJoin +(100) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(92) Scan parquet +(101) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(93) Filter +(102) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(94) Project +(103) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(95) Exchange +(104) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(105) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(97) Exchange +(106) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(99) Project +(108) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(100) Exchange +(109) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Scan parquet +(110) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(102) Filter +(111) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(103) Project +(112) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(104) Exchange +(113) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(106) Project +(115) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(107) Exchange +(116) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Scan parquet +(117) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(109) Filter +(118) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(110) Project +(119) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(111) Exchange +(120) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(121) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(113) Project +(122) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(114) HashAggregate +(123) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(115) Exchange +(124) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) HashAggregate +(125) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(117) TakeOrderedAndProject +(126) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(118) AdaptiveSparkPlan +(127) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt index 3c18ab436ed2..2a822968008a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt @@ -1,43 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (46) +AdaptiveSparkPlan (49) +- == Final Plan == - VeloxColumnarToRowExec (33) - +- ^ SortExecTransformer (31) - +- ^ InputIteratorTransformer (30) - +- ShuffleQueryStage (28), Statistics(X) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (45) - +- Exchange (44) - +- HashAggregate (43) - +- Exchange (42) - +- HashAggregate (41) - +- Project (40) - +- ShuffledHashJoin LeftAnti BuildRight (39) - :- Exchange (36) - : +- Filter (35) - : +- Scan parquet (34) - +- Exchange (38) - +- Scan parquet (37) + Sort (48) + +- Exchange (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- ShuffledHashJoin LeftAnti BuildRight (42) + :- Exchange (39) + : +- Filter (38) + : +- Scan parquet (37) + +- Exchange (41) + +- Scan parquet (40) (1) Scan parquet @@ -59,316 +62,328 @@ Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) Scan parquet +(10) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [1]: [o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [1]: [o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(26) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) ColumnarExchange +(29) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(30) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(28) ShuffleQueryStage +(31) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(29) InputAdapter +(32) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) InputIteratorTransformer +(33) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(31) SortExecTransformer +(34) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(32) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(33) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) Scan parquet +(37) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(35) Filter +(38) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(36) Exchange +(39) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(40) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(40) Project +(43) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(41) HashAggregate +(44) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(42) Exchange +(45) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) HashAggregate +(46) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(44) Exchange +(47) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(48) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(46) AdaptiveSparkPlan +(49) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (65) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ FilterExecTransformer (48) - +- ^ Scan parquet (47) + VeloxColumnarToRowExec (61) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- ^ FlushableHashAggregateExecTransformer (53) + +- ^ ProjectExecTransformer (52) + +- ^ FilterExecTransformer (51) + +- ^ Scan parquet (50) +- == Initial Plan == - HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (67) + +- Exchange (66) + +- HashAggregate (65) + +- Project (64) + +- Filter (63) + +- Scan parquet (62) -(47) Scan parquet +(50) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(48) FilterExecTransformer +(51) FilterExecTransformer Input [2]: [c_phone#X, c_acctbal#X] Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(49) ProjectExecTransformer +(52) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(50) FlushableHashAggregateExecTransformer +(53) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(51) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(52) ColumnarExchange +(55) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(56) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(54) InputAdapter +(57) InputAdapter Input [2]: [sum#X, count#X] -(55) InputIteratorTransformer +(58) InputIteratorTransformer Input [2]: [sum#X, count#X] -(56) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(58) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(59) Scan parquet +(62) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(60) Filter +(63) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(61) Project +(64) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(62) HashAggregate +(65) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(63) Exchange +(66) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(67) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(65) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (65) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ FilterExecTransformer (48) - +- ^ Scan parquet (47) + VeloxColumnarToRowExec (61) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- ^ FlushableHashAggregateExecTransformer (53) + +- ^ ProjectExecTransformer (52) + +- ^ FilterExecTransformer (51) + +- ^ Scan parquet (50) +- == Initial Plan == - HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) \ No newline at end of file + HashAggregate (67) + +- Exchange (66) + +- HashAggregate (65) + +- Project (64) + +- Filter (63) + +- Scan parquet (62) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt index eebd274d4709..51408f03e4e7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt @@ -1,56 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (49) - : +- Project (48) - : +- ShuffledHashJoin Inner BuildLeft (47) - : :- Exchange (43) - : : +- Project (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Exchange (53) - +- Project (52) - +- Filter (51) - +- Scan parquet (50) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Project (59) + +- ShuffledHashJoin Inner BuildRight (58) + :- Exchange (53) + : +- Project (52) + : +- ShuffledHashJoin Inner BuildLeft (51) + : :- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Exchange (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -72,244 +76,260 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(23) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(42) Project +(46) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(43) Exchange +(47) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(48) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(45) Filter +(49) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(46) Exchange +(50) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(51) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(48) Project +(52) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Exchange +(53) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(54) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(51) Filter +(55) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(52) Project +(56) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(53) Exchange +(57) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(58) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(55) Project +(59) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(56) HashAggregate +(60) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(57) HashAggregate +(61) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt index f05c9a5378c6..deba9b23f1fa 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt @@ -1,47 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftSemi BuildRight (43) - :- Exchange (38) - : +- Project (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftSemi BuildRight (46) + :- Exchange (41) + : +- Project (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -63,200 +66,212 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [l_orderkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [l_orderkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(35) Scan parquet +(38) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(36) Filter +(39) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(37) Project +(40) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(38) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(41) Project +(44) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(42) Exchange +(45) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(44) Project +(47) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(45) HashAggregate +(48) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(46) Exchange +(49) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(50) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(48) Exchange +(51) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(52) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt index 027ea4a926d2..257c7a998c7e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt @@ -1,115 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (134) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93), Statistics(X) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (133) - +- Exchange (132) - +- HashAggregate (131) - +- Exchange (130) - +- HashAggregate (129) - +- Project (128) - +- ShuffledHashJoin Inner BuildRight (127) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Project (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (126) - +- Project (125) - +- Filter (124) - +- Scan parquet (123) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Project (111) + : : : : +- Filter (110) + : : : : +- Scan parquet (109) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (137) + +- Project (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -131,552 +142,596 @@ Input [2]: [c_custkey#X, c_nationkey#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X + +(66) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: X + +(74) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [r_regionkey#X, r_name#X] Arguments: ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [1]: [r_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [1]: [r_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(99) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [2]: [n_name#X, revenue#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) Exchange +(108) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(109) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(99) Filter +(110) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(100) Project +(111) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(101) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(103) Project +(114) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(104) Exchange +(115) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(107) Exchange +(118) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(109) Project +(120) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(110) Exchange +(121) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(113) Exchange +(124) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(115) Project +(126) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(116) Exchange +(127) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(119) Exchange +(130) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(121) Project +(132) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(122) Exchange +(133) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(125) Project +(136) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(126) Exchange +(137) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(128) Project +(139) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(129) HashAggregate +(140) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(130) Exchange +(141) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) HashAggregate +(142) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(132) Exchange +(143) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(144) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(134) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/6.txt index 68854bdea473..3a3148c1c113 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnot Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt index eb5979fb5d84..1255a3a0c6ec 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt @@ -1,110 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (138) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ SortExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ShuffleQueryStage (70), Statistics(X) - +- ReusedExchange (69) + VeloxColumnarToRowExec (100) + +- ^ SortExecTransformer (98) + +- ^ InputIteratorTransformer (97) + +- ShuffleQueryStage (95), Statistics(X) + +- ColumnarExchange (94) + +- VeloxAppendBatches (93) + +- ^ RegularHashAggregateExecTransformer (91) + +- ^ InputIteratorTransformer (90) + +- ShuffleQueryStage (88), Statistics(X) + +- ColumnarExchange (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ReusedExchange (78) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (111) - : : +- Project (110) - : : +- ShuffledHashJoin Inner BuildRight (109) - : : :- Exchange (105) - : : : +- Project (104) - : : : +- ShuffledHashJoin Inner BuildRight (103) - : : : :- Exchange (99) - : : : : +- Project (98) - : : : : +- ShuffledHashJoin Inner BuildLeft (97) - : : : : :- Exchange (93) - : : : : : +- Filter (92) - : : : : : +- Scan parquet (91) - : : : : +- Exchange (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Exchange (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (114) - : +- Filter (113) - : +- Scan parquet (112) - +- Exchange (120) - +- Filter (119) - +- Scan parquet (118) + Sort (137) + +- Exchange (136) + +- HashAggregate (135) + +- Exchange (134) + +- HashAggregate (133) + +- Project (132) + +- ShuffledHashJoin Inner BuildRight (131) + :- Exchange (127) + : +- Project (126) + : +- ShuffledHashJoin Inner BuildRight (125) + : :- Exchange (121) + : : +- Project (120) + : : +- ShuffledHashJoin Inner BuildRight (119) + : : :- Exchange (115) + : : : +- Project (114) + : : : +- ShuffledHashJoin Inner BuildRight (113) + : : : :- Exchange (109) + : : : : +- Project (108) + : : : : +- ShuffledHashJoin Inner BuildLeft (107) + : : : : :- Exchange (103) + : : : : : +- Filter (102) + : : : : : +- Scan parquet (101) + : : : : +- Exchange (106) + : : : : +- Filter (105) + : : : : +- Scan parquet (104) + : : : +- Exchange (112) + : : : +- Filter (111) + : : : +- Scan parquet (110) + : : +- Exchange (118) + : : +- Filter (117) + : : +- Scan parquet (116) + : +- Exchange (124) + : +- Filter (123) + : +- Scan parquet (122) + +- Exchange (130) + +- Filter (129) + +- Scan parquet (128) (1) Scan parquet @@ -126,524 +136,564 @@ Input [2]: [s_suppkey#X, s_nationkey#X] Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(23) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [o_orderkey#X, o_custkey#X] Arguments: (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: X + +(40) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [c_custkey#X, c_nationkey#X] Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(69) ReusedExchange [Reuses operator id: 58] +(78) ReusedExchange [Reuses operator id: 66] Output [2]: [n_nationkey#X, n_name#X] -(70) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(71) InputAdapter +(80) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(72) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(73) ShuffledHashJoinExecTransformer +(82) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(74) ProjectExecTransformer +(83) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(75) FlushableHashAggregateExecTransformer +(84) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) ProjectExecTransformer +(85) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) WholeStageCodegenTransformer (X) +(86) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(78) ColumnarExchange +(87) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(88) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(80) InputAdapter +(89) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(81) InputIteratorTransformer +(90) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(82) RegularHashAggregateExecTransformer +(91) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(83) WholeStageCodegenTransformer (X) +(92) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(84) ColumnarExchange +(93) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(94) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(95) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(86) InputAdapter +(96) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(87) InputIteratorTransformer +(97) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(88) SortExecTransformer +(98) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(89) WholeStageCodegenTransformer (X) +(99) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(90) VeloxColumnarToRowExec +(100) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(91) Scan parquet +(101) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(102) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(103) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(104) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(105) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Exchange +(106) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(98) Project +(108) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(102) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(104) Project +(114) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(105) Exchange +(115) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(116) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(117) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(118) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(110) Project +(120) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(111) Exchange +(121) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(122) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(113) Filter +(123) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(114) Exchange +(124) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(116) Project +(126) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(117) Exchange +(127) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(128) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(119) Filter +(129) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(120) Exchange +(130) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(122) Project +(132) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(123) HashAggregate +(133) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(124) Exchange +(134) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(135) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(126) Exchange +(136) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(137) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(128) AdaptiveSparkPlan +(138) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt index 98bba133502c..8c66866aa476 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt @@ -1,150 +1,165 @@ == Physical Plan == -AdaptiveSparkPlan (177) +AdaptiveSparkPlan (192) +- == Final Plan == - VeloxColumnarToRowExec (125) - +- ^ SortExecTransformer (123) - +- ^ InputIteratorTransformer (122) - +- ShuffleQueryStage (120), Statistics(X) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ShuffleQueryStage (113), Statistics(X) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ShuffleQueryStage (96), Statistics(X) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ShuffleQueryStage (81), Statistics(X) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ShuffleQueryStage (66), Statistics(X) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ShuffleQueryStage (51), Statistics(X) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ShuffleQueryStage (36), Statistics(X) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ShuffleQueryStage (29), Statistics(X) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ FilterExecTransformer (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ShuffleQueryStage (44), Statistics(X) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ FilterExecTransformer (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ShuffleQueryStage (59), Statistics(X) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ FilterExecTransformer (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ShuffleQueryStage (74), Statistics(X) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ FilterExecTransformer (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ FilterExecTransformer (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ShuffleQueryStage (104), Statistics(X) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ FilterExecTransformer (100) - +- ^ Scan parquet (99) + VeloxColumnarToRowExec (140) + +- ^ SortExecTransformer (138) + +- ^ InputIteratorTransformer (137) + +- ShuffleQueryStage (135), Statistics(X) + +- ColumnarExchange (134) + +- VeloxAppendBatches (133) + +- ^ ProjectExecTransformer (131) + +- ^ RegularHashAggregateExecTransformer (130) + +- ^ InputIteratorTransformer (129) + +- ShuffleQueryStage (127), Statistics(X) + +- ColumnarExchange (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109), Statistics(X) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92), Statistics(X) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75), Statistics(X) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58), Statistics(X) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41), Statistics(X) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ FilterExecTransformer (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33), Statistics(X) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ FilterExecTransformer (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50), Statistics(X) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ FilterExecTransformer (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67), Statistics(X) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ FilterExecTransformer (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ FilterExecTransformer (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ FilterExecTransformer (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118), Statistics(X) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ FilterExecTransformer (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (176) - +- Exchange (175) - +- HashAggregate (174) - +- Exchange (173) - +- HashAggregate (172) - +- Project (171) - +- ShuffledHashJoin Inner BuildRight (170) - :- Exchange (165) - : +- Project (164) - : +- ShuffledHashJoin Inner BuildRight (163) - : :- Exchange (159) - : : +- Project (158) - : : +- ShuffledHashJoin Inner BuildRight (157) - : : :- Exchange (153) - : : : +- Project (152) - : : : +- ShuffledHashJoin Inner BuildRight (151) - : : : :- Exchange (147) - : : : : +- Project (146) - : : : : +- ShuffledHashJoin Inner BuildRight (145) - : : : : :- Exchange (141) - : : : : : +- Project (140) - : : : : : +- ShuffledHashJoin Inner BuildRight (139) - : : : : : :- Exchange (135) - : : : : : : +- Project (134) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) - : : : : : : :- Exchange (129) - : : : : : : : +- Project (128) - : : : : : : : +- Filter (127) - : : : : : : : +- Scan parquet (126) - : : : : : : +- Exchange (132) - : : : : : : +- Filter (131) - : : : : : : +- Scan parquet (130) - : : : : : +- Exchange (138) - : : : : : +- Filter (137) - : : : : : +- Scan parquet (136) - : : : : +- Exchange (144) - : : : : +- Filter (143) - : : : : +- Scan parquet (142) - : : : +- Exchange (150) - : : : +- Filter (149) - : : : +- Scan parquet (148) - : : +- Exchange (156) - : : +- Filter (155) - : : +- Scan parquet (154) - : +- Exchange (162) - : +- Filter (161) - : +- Scan parquet (160) - +- Exchange (169) - +- Project (168) - +- Filter (167) - +- Scan parquet (166) + Sort (191) + +- Exchange (190) + +- HashAggregate (189) + +- Exchange (188) + +- HashAggregate (187) + +- Project (186) + +- ShuffledHashJoin Inner BuildRight (185) + :- Exchange (180) + : +- Project (179) + : +- ShuffledHashJoin Inner BuildRight (178) + : :- Exchange (174) + : : +- Project (173) + : : +- ShuffledHashJoin Inner BuildRight (172) + : : :- Exchange (168) + : : : +- Project (167) + : : : +- ShuffledHashJoin Inner BuildRight (166) + : : : :- Exchange (162) + : : : : +- Project (161) + : : : : +- ShuffledHashJoin Inner BuildRight (160) + : : : : :- Exchange (156) + : : : : : +- Project (155) + : : : : : +- ShuffledHashJoin Inner BuildRight (154) + : : : : : :- Exchange (150) + : : : : : : +- Project (149) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (148) + : : : : : : :- Exchange (144) + : : : : : : : +- Project (143) + : : : : : : : +- Filter (142) + : : : : : : : +- Scan parquet (141) + : : : : : : +- Exchange (147) + : : : : : : +- Filter (146) + : : : : : : +- Scan parquet (145) + : : : : : +- Exchange (153) + : : : : : +- Filter (152) + : : : : : +- Scan parquet (151) + : : : : +- Exchange (159) + : : : : +- Filter (158) + : : : : +- Scan parquet (157) + : : : +- Exchange (165) + : : : +- Filter (164) + : : : +- Scan parquet (163) + : : +- Exchange (171) + : : +- Filter (170) + : : +- Scan parquet (169) + : +- Exchange (177) + : +- Filter (176) + : +- Scan parquet (175) + +- Exchange (184) + +- Project (183) + +- Filter (182) + +- Scan parquet (181) (1) Scan parquet @@ -166,732 +181,792 @@ Input [2]: [p_partkey#X, p_type#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [c_custkey#X, c_nationkey#X] Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(79) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(80) ColumnarExchange +(90) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: X + +(91) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(92) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(82) InputAdapter +(93) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(83) InputIteratorTransformer +(94) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(84) Scan parquet +(95) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) FilterExecTransformer +(96) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(86) ProjectExecTransformer +(97) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(92) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(93) ProjectExecTransformer +(105) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(94) WholeStageCodegenTransformer (X) +(106) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(95) ColumnarExchange +(107) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: X + +(108) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(109) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(97) InputAdapter +(110) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(98) InputIteratorTransformer +(111) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(99) Scan parquet +(112) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(100) FilterExecTransformer +(113) FilterExecTransformer Input [2]: [r_regionkey#X, r_name#X] Arguments: ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(101) ProjectExecTransformer +(114) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(102) WholeStageCodegenTransformer (X) +(115) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(103) ColumnarExchange +(116) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(117) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(104) ShuffleQueryStage +(118) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(105) InputAdapter +(119) InputAdapter Input [1]: [r_regionkey#X] -(106) InputIteratorTransformer +(120) InputIteratorTransformer Input [1]: [r_regionkey#X] -(107) ShuffledHashJoinExecTransformer +(121) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(108) ProjectExecTransformer +(122) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(109) FlushableHashAggregateExecTransformer +(123) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(110) ProjectExecTransformer +(124) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(111) WholeStageCodegenTransformer (X) +(125) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(112) ColumnarExchange +(126) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(113) ShuffleQueryStage +(127) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(114) InputAdapter +(128) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(115) InputIteratorTransformer +(129) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(116) RegularHashAggregateExecTransformer +(130) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(117) ProjectExecTransformer +(131) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(118) WholeStageCodegenTransformer (X) +(132) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(119) ColumnarExchange +(133) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(134) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(120) ShuffleQueryStage +(135) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(121) InputAdapter +(136) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(122) InputIteratorTransformer +(137) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(123) SortExecTransformer +(138) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(124) WholeStageCodegenTransformer (X) +(139) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(125) VeloxColumnarToRowExec +(140) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(126) Scan parquet +(141) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(127) Filter +(142) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(128) Project +(143) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(129) Exchange +(144) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) Scan parquet +(145) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(131) Filter +(146) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(132) Exchange +(147) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) ShuffledHashJoin +(148) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(134) Project +(149) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(135) Exchange +(150) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Scan parquet +(151) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(137) Filter +(152) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(138) Exchange +(153) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(154) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(140) Project +(155) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(141) Exchange +(156) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) Scan parquet +(157) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(143) Filter +(158) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(144) Exchange +(159) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) ShuffledHashJoin +(160) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(146) Project +(161) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(147) Exchange +(162) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Scan parquet +(163) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(149) Filter +(164) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(150) Exchange +(165) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) ShuffledHashJoin +(166) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(152) Project +(167) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(153) Exchange +(168) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) Scan parquet +(169) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(155) Filter +(170) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(156) Exchange +(171) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) ShuffledHashJoin +(172) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(158) Project +(173) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(159) Exchange +(174) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Scan parquet +(175) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(161) Filter +(176) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(162) Exchange +(177) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) ShuffledHashJoin +(178) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(164) Project +(179) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(165) Exchange +(180) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) Scan parquet +(181) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(167) Filter +(182) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(168) Project +(183) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(169) Exchange +(184) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) ShuffledHashJoin +(185) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(171) Project +(186) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(172) HashAggregate +(187) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(173) Exchange +(188) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(174) HashAggregate +(189) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(175) Exchange +(190) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(191) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(177) AdaptiveSparkPlan +(192) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt index 7f3917f80457..7435043d769d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt @@ -1,114 +1,125 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (144) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93), Statistics(X) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- HashAggregate (130) - +- Exchange (129) - +- HashAggregate (128) - +- Project (127) - +- ShuffledHashJoin Inner BuildRight (126) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (98) - : : : : : +- Project (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (125) - +- Filter (124) - +- Scan parquet (123) + Sort (143) + +- Exchange (142) + +- HashAggregate (141) + +- Exchange (140) + +- HashAggregate (139) + +- Project (138) + +- ShuffledHashJoin Inner BuildRight (137) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (109) + : : : : : +- Project (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -130,548 +141,592 @@ Input [2]: [p_partkey#X, p_name#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: X + +(57) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: isnotnull(o_orderkey#X) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: X + +(74) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(99) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(97) Project +(108) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(109) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(110) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(100) Filter +(111) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(101) Exchange +(112) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(103) Project +(114) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(115) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(118) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(109) Project +(120) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(121) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(113) Exchange +(124) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(115) Project +(126) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(116) Exchange +(127) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(119) Exchange +(130) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(121) Project +(132) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(122) Exchange +(133) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(125) Exchange +(136) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(137) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(127) Project +(138) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(128) HashAggregate +(139) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(129) Exchange +(140) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) HashAggregate +(141) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(131) Exchange +(142) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(143) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(133) AdaptiveSparkPlan +(144) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/1.txt index 090a9522f13a..468ad7c5d3aa 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt index 02cf374dd013..c740b49a1a8b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt @@ -1,78 +1,84 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- TakeOrderedAndProjectExecTransformer (59) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ShuffleQueryStage (36), Statistics(X) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ShuffleQueryStage (21), Statistics(X) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ShuffleQueryStage (29), Statistics(X) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ FilterExecTransformer (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (66) + +- TakeOrderedAndProjectExecTransformer (65) + +- ^ ProjectExecTransformer (63) + +- ^ RegularHashAggregateExecTransformer (62) + +- ^ InputIteratorTransformer (61) + +- ShuffleQueryStage (59), Statistics(X) + +- ColumnarExchange (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24), Statistics(X) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33), Statistics(X) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FilterExecTransformer (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- ShuffledHashJoin Inner BuildRight (81) - :- Exchange (77) - : +- Project (76) - : +- ShuffledHashJoin Inner BuildRight (75) - : :- Exchange (70) - : : +- Project (69) - : : +- ShuffledHashJoin Inner BuildRight (68) - : : :- Exchange (63) - : : : +- Filter (62) - : : : +- Scan parquet (61) - : : +- Exchange (67) - : : +- Project (66) - : : +- Filter (65) - : : +- Scan parquet (64) - : +- Exchange (74) - : +- Project (73) - : +- Filter (72) - : +- Scan parquet (71) - +- Exchange (80) - +- Filter (79) - +- Scan parquet (78) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- ShuffledHashJoin Inner BuildRight (87) + :- Exchange (83) + : +- Project (82) + : +- ShuffledHashJoin Inner BuildRight (81) + : :- Exchange (76) + : : +- Project (75) + : : +- ShuffledHashJoin Inner BuildRight (74) + : : :- Exchange (69) + : : : +- Filter (68) + : : : +- Scan parquet (67) + : : +- Exchange (73) + : : +- Project (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (80) + : +- Project (79) + : +- Filter (78) + : +- Scan parquet (77) + +- Exchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -94,364 +100,388 @@ Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acct Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: X + +(6) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Arguments: ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(55) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(56) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(58) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(59) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(60) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(61) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(62) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(57) ProjectExecTransformer +(63) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(58) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(59) TakeOrderedAndProjectExecTransformer +(65) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(60) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(61) Scan parquet +(67) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) Filter +(68) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(63) Exchange +(69) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(65) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(66) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(67) Exchange +(73) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) ShuffledHashJoin +(74) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(69) Project +(75) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(70) Exchange +(76) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(77) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(72) Filter +(78) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(73) Project +(79) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(74) Exchange +(80) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(81) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(76) Project +(82) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(77) Exchange +(83) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(80) Exchange +(86) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(87) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(82) Project +(88) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(89) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(84) Exchange +(90) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(91) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(86) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt index b14866370aab..89dd19ffe0ce 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt @@ -1,65 +1,70 @@ == Physical Plan == -AdaptiveSparkPlan (72) +AdaptiveSparkPlan (77) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ SortExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ShuffleQueryStage (38), Statistics(X) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (55) + +- ^ SortExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ FilterExecTransformer (46) + +- ^ RegularHashAggregateExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42), Statistics(X) + +- ColumnarExchange (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (71) - +- Exchange (70) - +- Filter (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- ShuffledHashJoin Inner BuildRight (64) - :- Exchange (59) - : +- Project (58) - : +- ShuffledHashJoin Inner BuildRight (57) - : :- Exchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- Exchange (56) - : +- Filter (55) - : +- Scan parquet (54) - +- Exchange (63) - +- Project (62) - +- Filter (61) - +- Scan parquet (60) + Sort (76) + +- Exchange (75) + +- Filter (74) + +- HashAggregate (73) + +- Exchange (72) + +- HashAggregate (71) + +- Project (70) + +- ShuffledHashJoin Inner BuildRight (69) + :- Exchange (64) + : +- Project (63) + : +- ShuffledHashJoin Inner BuildRight (62) + : :- Exchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- Exchange (61) + : +- Filter (60) + : +- Scan parquet (59) + +- Exchange (68) + +- Project (67) + +- Filter (66) + +- Scan parquet (65) (1) Scan parquet @@ -81,557 +86,587 @@ Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [1]: [n_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [n_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(34) FlushableHashAggregateExecTransformer +(38) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(41) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(43) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(45) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(42) FilterExecTransformer +(46) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(43) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(44) ColumnarExchange +(48) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(49) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(46) InputAdapter +(51) InputAdapter Input [2]: [ps_partkey#X, value#X] -(47) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(48) SortExecTransformer +(53) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(49) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(50) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(51) Scan parquet +(56) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(52) Filter +(57) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(53) Exchange +(58) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(59) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(60) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) Exchange +(61) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(62) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(58) Project +(63) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(59) Exchange +(64) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(65) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(61) Filter +(66) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(62) Project +(67) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(63) Exchange +(68) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(69) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(65) Project +(70) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(66) HashAggregate +(71) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(67) Exchange +(72) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(73) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(69) Filter +(74) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(70) Exchange +(75) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(76) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(72) AdaptiveSparkPlan +(77) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (120) +Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (127) +- == Final Plan == - VeloxColumnarToRowExec (102) - +- ^ ProjectExecTransformer (100) - +- ^ RegularHashAggregateExecTransformer (99) - +- ^ RegularHashAggregateExecTransformer (98) - +- ^ ProjectExecTransformer (97) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) - :- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) - : :- ^ InputIteratorTransformer (80) - : : +- ShuffleQueryStage (78), Statistics(X) - : : +- ColumnarExchange (77) - : : +- ^ ProjectExecTransformer (75) - : : +- ^ FilterExecTransformer (74) - : : +- ^ Scan parquet (73) - : +- ^ InputIteratorTransformer (84) - : +- ShuffleQueryStage (82), Statistics(X) - : +- ReusedExchange (81) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ReusedExchange (92) + VeloxColumnarToRowExec (109) + +- ^ ProjectExecTransformer (107) + +- ^ RegularHashAggregateExecTransformer (106) + +- ^ RegularHashAggregateExecTransformer (105) + +- ^ ProjectExecTransformer (104) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (103) + :- ^ InputIteratorTransformer (98) + : +- ShuffleQueryStage (96), Statistics(X) + : +- ColumnarExchange (95) + : +- VeloxAppendBatches (94) + : +- ^ ProjectExecTransformer (92) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (91) + : :- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ FilterExecTransformer (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (90) + : +- ShuffleQueryStage (88), Statistics(X) + : +- ReusedExchange (87) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ReusedExchange (99) +- == Initial Plan == - HashAggregate (119) - +- HashAggregate (118) - +- Project (117) - +- ShuffledHashJoin Inner BuildRight (116) - :- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildRight (109) - : :- Exchange (105) - : : +- Filter (104) - : : +- Scan parquet (103) - : +- Exchange (108) - : +- Filter (107) - : +- Scan parquet (106) - +- Exchange (115) - +- Project (114) - +- Filter (113) - +- Scan parquet (112) - - -(73) Scan parquet + HashAggregate (126) + +- HashAggregate (125) + +- Project (124) + +- ShuffledHashJoin Inner BuildRight (123) + :- Exchange (118) + : +- Project (117) + : +- ShuffledHashJoin Inner BuildRight (116) + : :- Exchange (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- Exchange (115) + : +- Filter (114) + : +- Scan parquet (113) + +- Exchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) + + +(78) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(74) FilterExecTransformer +(79) FilterExecTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: isnotnull(ps_suppkey#X) -(75) ProjectExecTransformer +(80) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(76) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(77) ColumnarExchange +(82) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(83) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(78) ShuffleQueryStage +(84) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(79) InputAdapter +(85) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(80) InputIteratorTransformer +(86) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(81) ReusedExchange [Reuses operator id: 13] +(87) ReusedExchange [Reuses operator id: 15] Output [2]: [s_suppkey#X, s_nationkey#X] -(82) ShuffleQueryStage +(88) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(83) InputAdapter +(89) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(84) InputIteratorTransformer +(90) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(85) ShuffledHashJoinExecTransformer +(91) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(86) ProjectExecTransformer +(92) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(87) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(88) ColumnarExchange +(94) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(95) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(96) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(90) InputAdapter +(97) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(91) InputIteratorTransformer +(98) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(92) ReusedExchange [Reuses operator id: 28] +(99) ReusedExchange [Reuses operator id: 32] Output [1]: [n_nationkey#X] -(93) ShuffleQueryStage +(100) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(94) InputAdapter +(101) InputAdapter Input [1]: [n_nationkey#X] -(95) InputIteratorTransformer +(102) InputIteratorTransformer Input [1]: [n_nationkey#X] -(96) ShuffledHashJoinExecTransformer +(103) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(97) ProjectExecTransformer +(104) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(98) RegularHashAggregateExecTransformer +(105) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(99) RegularHashAggregateExecTransformer +(106) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(100) ProjectExecTransformer +(107) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(101) WholeStageCodegenTransformer (X) +(108) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(102) VeloxColumnarToRowExec +(109) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(103) Scan parquet +(110) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(104) Filter +(111) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(105) Exchange +(112) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(113) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(107) Filter +(114) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(108) Exchange +(115) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(116) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(110) Project +(117) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(111) Exchange +(118) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(119) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(113) Filter +(120) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(114) Project +(121) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(115) Exchange +(122) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(123) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(117) Project +(124) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(118) HashAggregate +(125) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(119) HashAggregate +(126) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(120) AdaptiveSparkPlan +(127) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt index 27765da815dd..75948ca7ad8b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt @@ -1,46 +1,49 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin Inner BuildLeft (42) - :- Exchange (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- ShuffledHashJoin Inner BuildLeft (45) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -62,198 +65,210 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Arguments: ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(31) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) Scan parquet +(38) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(36) Filter +(39) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(37) Exchange +(40) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(41) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(40) Project +(43) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(41) Exchange +(44) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(45) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(43) Project +(46) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(44) HashAggregate +(47) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(45) Exchange +(48) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(49) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(47) Exchange +(50) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(51) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt index 79b9dcd18bd1..5a8bd7bbc955 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt @@ -1,49 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftOuter BuildRight (43) - :- Exchange (38) - : +- Scan parquet (37) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftOuter BuildRight (46) + :- Exchange (41) + : +- Scan parquet (40) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -60,225 +63,237 @@ Input [1]: [c_custkey#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(4) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) FilterExecTransformer +(10) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Arguments: ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(44) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) Exchange +(45) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(44) Project +(47) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(48) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt index 4db67eada562..c6f425f00868 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (23) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (25) + +- ^ ProjectExecTransformer (23) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (34) - +- HashAggregate (33) - +- Project (32) - +- ShuffledHashJoin Inner BuildRight (31) - :- Exchange (27) - : +- Project (26) - : +- Filter (25) - : +- Scan parquet (24) - +- Exchange (30) - +- Filter (29) - +- Scan parquet (28) + HashAggregate (36) + +- HashAggregate (35) + +- Project (34) + +- ShuffledHashJoin Inner BuildRight (33) + :- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -52,146 +54,154 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [2]: [p_partkey#X, p_type#X] Arguments: isnotnull(p_partkey#X) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(21) ProjectExecTransformer +(23) ProjectExecTransformer Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(22) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(23) VeloxColumnarToRowExec +(25) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(24) Scan parquet +(26) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(26) Project +(28) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) Exchange +(29) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Scan parquet +(30) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(30) Exchange +(32) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) ShuffledHashJoin +(33) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(32) Project +(34) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(34) HashAggregate +(36) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] -(35) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt index c8f4e2c84ac3..5592b9e73add 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt @@ -1,43 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- AQEShuffleRead (26) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (29) + +- AQEShuffleRead (28) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (22) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- Project (39) - +- ShuffledHashJoin Inner BuildLeft (38) - :- Exchange (30) - : +- Filter (29) - : +- Scan parquet (28) - +- Filter (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (43) + +- Exchange (42) + +- Project (41) + +- ShuffledHashJoin Inner BuildLeft (40) + :- Exchange (32) + : +- Filter (31) + : +- Scan parquet (30) + +- Filter (39) + +- HashAggregate (38) + +- Exchange (37) + +- HashAggregate (36) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -59,330 +61,338 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(16) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(20) FilterExecTransformer +(21) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(21) ShuffledHashJoinExecTransformer +(22) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(26) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) AQEShuffleRead +(28) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(27) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) Scan parquet +(30) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(30) Exchange +(32) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) Scan parquet +(33) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(32) Filter +(34) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(33) Project +(35) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(34) HashAggregate +(36) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(35) Exchange +(37) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(38) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(37) Filter +(39) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(38) ShuffledHashJoin +(40) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(39) Project +(41) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(40) Exchange +(42) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(43) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (67) +Subquery:1 Hosting operator id = 21 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ RegularHashAggregateExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ ProjectExecTransformer (47) - +- ^ FlushableHashAggregateExecTransformer (46) - +- ^ ProjectExecTransformer (45) - +- ^ FilterExecTransformer (44) - +- ^ Scan parquet (43) + VeloxColumnarToRowExec (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ ProjectExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ InputIteratorTransformer (54) + +- ShuffleQueryStage (52), Statistics(X) + +- ColumnarExchange (51) + +- ^ ProjectExecTransformer (49) + +- ^ FlushableHashAggregateExecTransformer (48) + +- ^ ProjectExecTransformer (47) + +- ^ FilterExecTransformer (46) + +- ^ Scan parquet (45) +- == Initial Plan == - HashAggregate (66) - +- HashAggregate (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (68) + +- HashAggregate (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- Filter (62) + +- Scan parquet (61) -(43) Scan parquet +(45) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(44) FilterExecTransformer +(46) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(45) ProjectExecTransformer +(47) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) FlushableHashAggregateExecTransformer +(48) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(48) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(49) ColumnarExchange +(51) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(51) InputAdapter +(53) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(53) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(54) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(55) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(56) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(57) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(58) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(59) Scan parquet +(61) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(60) Filter +(62) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(61) Project +(63) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(63) Exchange +(65) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(65) HashAggregate +(67) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(66) HashAggregate +(68) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(67) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt index 247b853fb1c4..73703d5baf67 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt @@ -1,59 +1,62 @@ == Physical Plan == -AdaptiveSparkPlan (64) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (42) - +- ^ SortExecTransformer (40) - +- ^ InputIteratorTransformer (39) - +- ShuffleQueryStage (37), Statistics(X) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (45) + +- ^ SortExecTransformer (43) + +- ^ InputIteratorTransformer (42) + +- ShuffleQueryStage (40), Statistics(X) + +- ColumnarExchange (39) + +- VeloxAppendBatches (38) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- ^ ProjectExecTransformer (30) + +- ^ FlushableHashAggregateExecTransformer (29) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (63) - +- Exchange (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- HashAggregate (58) - +- Exchange (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (50) - : +- BroadcastHashJoin LeftAnti BuildRight (49) - : :- Filter (44) - : : +- Scan parquet (43) - : +- BroadcastExchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (53) - +- Filter (52) - +- Scan parquet (51) + Sort (66) + +- Exchange (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- HashAggregate (61) + +- Exchange (60) + +- HashAggregate (59) + +- Project (58) + +- ShuffledHashJoin Inner BuildRight (57) + :- Exchange (53) + : +- BroadcastHashJoin LeftAnti BuildRight (52) + : :- Filter (47) + : : +- Scan parquet (46) + : +- BroadcastExchange (51) + : +- Project (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -75,273 +78,285 @@ Input [2]: [ps_partkey#X, ps_suppkey#X] Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) FlushableHashAggregateExecTransformer +(29) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(30) ColumnarExchange +(32) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(32) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) ColumnarExchange +(38) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(39) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(40) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(38) InputAdapter +(41) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) InputIteratorTransformer +(42) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(40) SortExecTransformer +(43) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(41) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(42) VeloxColumnarToRowExec +(45) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) Scan parquet +(46) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(45) Scan parquet +(48) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(46) Filter +(49) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(47) Project +(50) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(48) BroadcastExchange +(51) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(49) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: LeftAnti Join condition: None -(50) Exchange +(53) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(54) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(53) Exchange +(56) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(57) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(55) Project +(58) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(56) HashAggregate +(59) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) HashAggregate +(61) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(59) HashAggregate +(62) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(60) Exchange +(63) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(62) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) Sort +(66) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(64) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt index fad48184fed1..081d1e972014 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt @@ -1,56 +1,58 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ ProjectExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) - :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ FilterExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ FilterExecTransformer (20) - +- ^ Scan parquet (19) + VeloxColumnarToRowExec (39) + +- ^ ProjectExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ ProjectExecTransformer (34) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (33) + :- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ FilterExecTransformer (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (56) - +- HashAggregate (55) - +- Project (54) - +- ShuffledHashJoin Inner BuildRight (53) - :- Project (46) - : +- ShuffledHashJoin Inner BuildRight (45) - : :- Exchange (40) - : : +- Filter (39) - : : +- Scan parquet (38) - : +- Exchange (44) - : +- Project (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Filter (48) - +- Scan parquet (47) + HashAggregate (58) + +- HashAggregate (57) + +- Project (56) + +- ShuffledHashJoin Inner BuildRight (55) + :- Project (48) + : +- ShuffledHashJoin Inner BuildRight (47) + : :- Exchange (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Exchange (46) + : +- Project (45) + : +- Filter (44) + : +- Scan parquet (43) + +- Filter (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Filter (50) + +- Scan parquet (49) (1) Scan parquet @@ -72,254 +74,262 @@ Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [p_partkey#X, p_brand#X, p_container#X] Arguments: ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(19) Scan parquet +(21) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(20) FilterExecTransformer +(22) FilterExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Arguments: isnotnull(l_partkey#X) -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(30) FilterExecTransformer +(32) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(31) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(32) ProjectExecTransformer +(34) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(33) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(35) ProjectExecTransformer +(37) ProjectExecTransformer Output [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(36) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(37) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(38) Scan parquet +(40) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(39) Filter +(41) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(40) Exchange +(42) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(43) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(43) Project +(45) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(44) Exchange +(46) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(46) Project +(48) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(47) Scan parquet +(49) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(48) Filter +(50) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(49) HashAggregate +(51) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(50) Exchange +(52) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) Filter +(54) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(53) ShuffledHashJoin +(55) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(54) Project +(56) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) HashAggregate +(57) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(56) HashAggregate +(58) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] -(57) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt index 5a6f04064349..8c55d747d0ae 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt @@ -1,89 +1,94 @@ == Physical Plan == -AdaptiveSparkPlan (97) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- TakeOrderedAndProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) - :- ^ InputIteratorTransformer (41) - : +- ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ShuffleQueryStage (22), Statistics(X) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) - :- ^ InputIteratorTransformer (49) - : +- ShuffleQueryStage (47), Statistics(X) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ FilterExecTransformer (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ShuffleQueryStage (51), Statistics(X) - +- ReusedExchange (50) + VeloxColumnarToRowExec (69) + +- TakeOrderedAndProjectExecTransformer (68) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ RegularHashAggregateExecTransformer (65) + +- ^ ProjectExecTransformer (64) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (63) + :- ^ InputIteratorTransformer (45) + : +- ShuffleQueryStage (43), Statistics(X) + : +- ColumnarExchange (42) + : +- VeloxAppendBatches (41) + : +- ^ ProjectExecTransformer (39) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (38) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (37) + : +- ShuffleQueryStage (35), Statistics(X) + : +- ColumnarExchange (34) + : +- VeloxAppendBatches (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (30) + : :- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ Scan parquet (10) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ RegularHashAggregateExecTransformer (27) + : +- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) + :- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- VeloxAppendBatches (50) + : +- ^ ProjectExecTransformer (48) + : +- ^ FilterExecTransformer (47) + : +- ^ Scan parquet (46) + +- ^ ProjectExecTransformer (61) + +- ^ FilterExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ReusedExchange (55) +- == Initial Plan == - TakeOrderedAndProject (96) - +- HashAggregate (95) - +- HashAggregate (94) - +- Project (93) - +- ShuffledHashJoin Inner BuildRight (92) - :- Exchange (81) - : +- Project (80) - : +- ShuffledHashJoin Inner BuildLeft (79) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (78) - : +- ShuffledHashJoin LeftSemi BuildRight (77) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Project (76) - : +- Filter (75) - : +- HashAggregate (74) - : +- Exchange (73) - : +- HashAggregate (72) - : +- Scan parquet (71) - +- ShuffledHashJoin LeftSemi BuildRight (91) - :- Exchange (84) - : +- Filter (83) - : +- Scan parquet (82) - +- Project (90) - +- Filter (89) - +- HashAggregate (88) - +- Exchange (87) - +- HashAggregate (86) - +- Scan parquet (85) + TakeOrderedAndProject (101) + +- HashAggregate (100) + +- HashAggregate (99) + +- Project (98) + +- ShuffledHashJoin Inner BuildRight (97) + :- Exchange (86) + : +- Project (85) + : +- ShuffledHashJoin Inner BuildLeft (84) + : :- Exchange (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (83) + : +- ShuffledHashJoin LeftSemi BuildRight (82) + : :- Exchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- Project (81) + : +- Filter (80) + : +- HashAggregate (79) + : +- Exchange (78) + : +- HashAggregate (77) + : +- Scan parquet (76) + +- ShuffledHashJoin LeftSemi BuildRight (96) + :- Exchange (89) + : +- Filter (88) + : +- Scan parquet (87) + +- Project (95) + +- Filter (94) + +- HashAggregate (93) + +- Exchange (92) + +- HashAggregate (91) + +- Scan parquet (90) (1) Scan parquet @@ -105,428 +110,448 @@ Input [2]: [c_custkey#X, c_name#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(26) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(27) ProjectExecTransformer +(29) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(28) ShuffledHashJoinExecTransformer +(30) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(30) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(31) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(33) InputAdapter +(36) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(34) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) ShuffledHashJoinExecTransformer +(38) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(36) ProjectExecTransformer +(39) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(38) ColumnarExchange +(41) VeloxAppendBatches +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(42) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(43) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(40) InputAdapter +(44) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(41) InputIteratorTransformer +(45) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(42) Scan parquet +(46) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(43) FilterExecTransformer +(47) FilterExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Arguments: isnotnull(l_orderkey#X) -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(46) ColumnarExchange +(50) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: X + +(51) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(52) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(48) InputAdapter +(53) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(49) InputIteratorTransformer +(54) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(50) ReusedExchange [Reuses operator id: 21] +(55) ReusedExchange [Reuses operator id: 23] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) ShuffleQueryStage +(56) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(52) InputAdapter +(57) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(53) InputIteratorTransformer +(58) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(54) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(55) FilterExecTransformer +(60) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(56) ProjectExecTransformer +(61) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(58) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(59) ProjectExecTransformer +(64) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(60) RegularHashAggregateExecTransformer +(65) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(61) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(62) WholeStageCodegenTransformer (X) +(67) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(63) TakeOrderedAndProjectExecTransformer +(68) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(64) VeloxColumnarToRowExec +(69) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(65) Scan parquet +(70) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(66) Filter +(71) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(67) Exchange +(72) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(73) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(69) Filter +(74) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(70) Exchange +(75) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(76) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(77) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(78) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(79) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(80) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(81) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(78) Exchange +(83) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) ShuffledHashJoin +(84) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(80) Project +(85) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(81) Exchange +(86) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(87) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(83) Filter +(88) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(84) Exchange +(89) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(90) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(86) HashAggregate +(91) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(87) Exchange +(92) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) HashAggregate +(93) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(89) Filter +(94) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(90) Project +(95) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(91) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(93) Project +(98) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(94) HashAggregate +(99) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(95) HashAggregate +(100) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(96) TakeOrderedAndProject +(101) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(97) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt index 2f7bb7995dd9..baf4b2a51607 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (33) - +- HashAggregate (32) - +- Project (31) - +- ShuffledHashJoin Inner BuildRight (30) - :- Exchange (26) - : +- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- Exchange (29) - +- Filter (28) - +- Scan parquet (27) + HashAggregate (35) + +- HashAggregate (34) + +- Project (33) + +- ShuffledHashJoin Inner BuildRight (32) + :- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) (1) Scan parquet @@ -51,142 +53,150 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(24) VeloxColumnarToRowExec Input [1]: [revenue#X] -(23) Scan parquet +(25) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(24) Filter +(26) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(25) Project +(27) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(26) Exchange +(28) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Scan parquet +(29) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(28) Filter +(30) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(29) Exchange +(31) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) ShuffledHashJoin +(32) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(31) Project +(33) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(32) HashAggregate +(34) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(34) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt index e8aa97a29e7a..7ddecfe855eb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt @@ -1,109 +1,119 @@ == Physical Plan == -AdaptiveSparkPlan (123) +AdaptiveSparkPlan (133) +- == Final Plan == - VeloxColumnarToRowExec (83) - +- AQEShuffleRead (82) - +- ShuffleQueryStage (81), Statistics(X) - +- ColumnarExchange (80) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ShuffleQueryStage (29), Statistics(X) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ShuffleQueryStage (14), Statistics(X) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ FilterExecTransformer (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ShuffleQueryStage (22), Statistics(X) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ FilterExecTransformer (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ FilterExecTransformer (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41), Statistics(X) - : +- ReusedExchange (40) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (93) + +- AQEShuffleRead (92) + +- ShuffleQueryStage (91), Statistics(X) + +- ColumnarExchange (90) + +- VeloxAppendBatches (89) + +- ^ ProjectExecTransformer (87) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (86) + :- ^ InputIteratorTransformer (76) + : +- ShuffleQueryStage (74), Statistics(X) + : +- ColumnarExchange (73) + : +- VeloxAppendBatches (72) + : +- ^ ProjectExecTransformer (70) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (69) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- VeloxAppendBatches (64) + : +- ^ ProjectExecTransformer (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (61) + : :- ^ InputIteratorTransformer (35) + : : +- ShuffleQueryStage (33), Statistics(X) + : : +- ColumnarExchange (32) + : : +- VeloxAppendBatches (31) + : : +- ^ ProjectExecTransformer (29) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : : :- ^ InputIteratorTransformer (18) + : : : +- ShuffleQueryStage (16), Statistics(X) + : : : +- ColumnarExchange (15) + : : : +- VeloxAppendBatches (14) + : : : +- ^ ProjectExecTransformer (12) + : : : +- ^ FilterExecTransformer (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (27) + : : +- ShuffleQueryStage (25), Statistics(X) + : : +- ColumnarExchange (24) + : : +- VeloxAppendBatches (23) + : : +- ^ ProjectExecTransformer (21) + : : +- ^ FilterExecTransformer (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (60) + : +- ShuffleQueryStage (58), Statistics(X) + : +- ColumnarExchange (57) + : +- VeloxAppendBatches (56) + : +- ^ ProjectExecTransformer (54) + : +- ^ FilterExecTransformer (53) + : +- ^ ProjectExecTransformer (52) + : +- ^ RegularHashAggregateExecTransformer (51) + : +- ^ RegularHashAggregateExecTransformer (50) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (49) + : :- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ FilterExecTransformer (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (48) + : +- ShuffleQueryStage (46), Statistics(X) + : +- ReusedExchange (45) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- VeloxAppendBatches (81) + +- ^ ProjectExecTransformer (79) + +- ^ FilterExecTransformer (78) + +- ^ Scan parquet (77) +- == Initial Plan == - Sort (122) - +- Exchange (121) - +- Project (120) - +- ShuffledHashJoin Inner BuildRight (119) - :- Exchange (114) - : +- Project (113) - : +- ShuffledHashJoin LeftSemi BuildRight (112) - : :- Exchange (86) - : : +- Filter (85) - : : +- Scan parquet (84) - : +- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildLeft (109) - : :- Exchange (95) - : : +- ShuffledHashJoin LeftSemi BuildRight (94) - : : :- Exchange (89) - : : : +- Filter (88) - : : : +- Scan parquet (87) - : : +- Exchange (93) - : : +- Project (92) - : : +- Filter (91) - : : +- Scan parquet (90) - : +- Exchange (108) - : +- Filter (107) - : +- HashAggregate (106) - : +- HashAggregate (105) - : +- ShuffledHashJoin LeftSemi BuildRight (104) - : :- Exchange (99) - : : +- Project (98) - : : +- Filter (97) - : : +- Scan parquet (96) - : +- Exchange (103) - : +- Project (102) - : +- Filter (101) - : +- Scan parquet (100) - +- Exchange (118) - +- Project (117) - +- Filter (116) - +- Scan parquet (115) + Sort (132) + +- Exchange (131) + +- Project (130) + +- ShuffledHashJoin Inner BuildRight (129) + :- Exchange (124) + : +- Project (123) + : +- ShuffledHashJoin LeftSemi BuildRight (122) + : :- Exchange (96) + : : +- Filter (95) + : : +- Scan parquet (94) + : +- Exchange (121) + : +- Project (120) + : +- ShuffledHashJoin Inner BuildLeft (119) + : :- Exchange (105) + : : +- ShuffledHashJoin LeftSemi BuildRight (104) + : : :- Exchange (99) + : : : +- Filter (98) + : : : +- Scan parquet (97) + : : +- Exchange (103) + : : +- Project (102) + : : +- Filter (101) + : : +- Scan parquet (100) + : +- Exchange (118) + : +- Filter (117) + : +- HashAggregate (116) + : +- HashAggregate (115) + : +- ShuffledHashJoin LeftSemi BuildRight (114) + : :- Exchange (109) + : : +- Project (108) + : : +- Filter (107) + : : +- Scan parquet (106) + : +- Exchange (113) + : +- Project (112) + : +- Filter (111) + : +- Scan parquet (110) + +- Exchange (128) + +- Project (127) + +- Filter (126) + +- Scan parquet (125) (1) Scan parquet @@ -125,518 +135,558 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(18) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [p_partkey#X, p_name#X] Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [1]: [p_partkey#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [1]: [p_partkey#X] -(25) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(32) Scan parquet +(36) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(33) FilterExecTransformer +(37) FilterExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(40) ReusedExchange [Reuses operator id: 21] +(45) ReusedExchange [Reuses operator id: 24] Output [1]: [p_partkey#X] -(41) ShuffleQueryStage +(46) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(42) InputAdapter +(47) InputAdapter Input [1]: [p_partkey#X] -(43) InputIteratorTransformer +(48) InputIteratorTransformer Input [1]: [p_partkey#X] -(44) ShuffledHashJoinExecTransformer +(49) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(45) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(46) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(47) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(48) FilterExecTransformer +(53) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(49) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(51) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(53) InputAdapter +(59) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(54) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(55) ShuffledHashJoinExecTransformer +(61) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(58) ColumnarExchange +(64) VeloxAppendBatches +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: X + +(65) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(66) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(60) InputAdapter +(67) InputAdapter Input [1]: [ps_suppkey#X] -(61) InputIteratorTransformer +(68) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(62) ShuffledHashJoinExecTransformer +(69) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(63) ProjectExecTransformer +(70) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(71) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(72) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(73) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(74) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(75) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(68) InputIteratorTransformer +(76) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(69) Scan parquet +(77) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer +(78) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(71) ProjectExecTransformer +(79) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(73) ColumnarExchange +(81) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(82) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(83) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(75) InputAdapter +(84) InputAdapter Input [1]: [n_nationkey#X] -(76) InputIteratorTransformer +(85) InputIteratorTransformer Input [1]: [n_nationkey#X] -(77) ShuffledHashJoinExecTransformer +(86) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(87) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(79) WholeStageCodegenTransformer (X) +(88) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(80) ColumnarExchange +(89) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(90) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(91) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(82) AQEShuffleRead +(92) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(83) VeloxColumnarToRowExec +(93) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(84) Scan parquet +(94) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(85) Filter +(95) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(86) Exchange +(96) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Scan parquet +(97) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(88) Filter +(98) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(89) Exchange +(99) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(100) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(91) Filter +(101) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(92) Project +(102) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(93) Exchange +(103) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) ShuffledHashJoin +(104) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(95) Exchange +(105) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) Scan parquet +(106) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(97) Filter +(107) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(98) Project +(108) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(102) Project +(112) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(103) Exchange +(113) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(105) HashAggregate +(115) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(106) HashAggregate +(116) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(107) Filter +(117) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(108) Exchange +(118) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(110) Project +(120) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(111) Exchange +(121) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(113) Project +(123) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(114) Exchange +(124) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) Scan parquet +(125) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(116) Filter +(126) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(117) Project +(127) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(118) Exchange +(128) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(129) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(120) Project +(130) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(121) Exchange +(131) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Sort +(132) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(123) AdaptiveSparkPlan +(133) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt index 323db548d24e..ede94728ceb2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt @@ -1,103 +1,112 @@ == Physical Plan == -AdaptiveSparkPlan (118) +AdaptiveSparkPlan (127) +- == Final Plan == - VeloxColumnarToRowExec (81) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ShuffleQueryStage (21), Statistics(X) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ShuffleQueryStage (30), Statistics(X) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ FilterExecTransformer (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ShuffleQueryStage (67), Statistics(X) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ FilterExecTransformer (63) - +- ^ Scan parquet (62) + VeloxColumnarToRowExec (90) + +- ^ RegularHashAggregateExecTransformer (88) + +- ^ InputIteratorTransformer (87) + +- ShuffleQueryStage (85), Statistics(X) + +- ColumnarExchange (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24), Statistics(X) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34), Statistics(X) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ FilterExecTransformer (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ FilterExecTransformer (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ FilterExecTransformer (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (117) - +- HashAggregate (116) - +- Exchange (115) - +- HashAggregate (114) - +- Project (113) - +- ShuffledHashJoin Inner BuildRight (112) - :- Exchange (107) - : +- Project (106) - : +- ShuffledHashJoin Inner BuildRight (105) - : :- Exchange (100) - : : +- Project (99) - : : +- ShuffledHashJoin Inner BuildLeft (98) - : : :- Exchange (84) - : : : +- Filter (83) - : : : +- Scan parquet (82) - : : +- Exchange (97) - : : +- ShuffledHashJoin LeftAnti BuildRight (96) - : : :- ShuffledHashJoin LeftSemi BuildRight (91) - : : : :- Exchange (88) - : : : : +- Project (87) - : : : : +- Filter (86) - : : : : +- Scan parquet (85) - : : : +- Exchange (90) - : : : +- Scan parquet (89) - : : +- Exchange (95) - : : +- Project (94) - : : +- Filter (93) - : : +- Scan parquet (92) - : +- Exchange (104) - : +- Project (103) - : +- Filter (102) - : +- Scan parquet (101) - +- Exchange (111) - +- Project (110) - +- Filter (109) - +- Scan parquet (108) + TakeOrderedAndProject (126) + +- HashAggregate (125) + +- Exchange (124) + +- HashAggregate (123) + +- Project (122) + +- ShuffledHashJoin Inner BuildRight (121) + :- Exchange (116) + : +- Project (115) + : +- ShuffledHashJoin Inner BuildRight (114) + : :- Exchange (109) + : : +- Project (108) + : : +- ShuffledHashJoin Inner BuildLeft (107) + : : :- Exchange (93) + : : : +- Filter (92) + : : : +- Scan parquet (91) + : : +- Exchange (106) + : : +- ShuffledHashJoin LeftAnti BuildRight (105) + : : :- ShuffledHashJoin LeftSemi BuildRight (100) + : : : :- Exchange (97) + : : : : +- Project (96) + : : : : +- Filter (95) + : : : : +- Scan parquet (94) + : : : +- Exchange (99) + : : : +- Scan parquet (98) + : : +- Exchange (104) + : : +- Project (103) + : : +- Filter (102) + : : +- Scan parquet (101) + : +- Exchange (113) + : +- Project (112) + : +- Filter (111) + : +- Scan parquet (110) + +- Exchange (120) + +- Project (119) + +- Filter (118) + +- Scan parquet (117) (1) Scan parquet @@ -119,500 +128,536 @@ Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(24) ShuffledHashJoinExecTransformer +(27) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(25) Scan parquet +(28) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(26) FilterExecTransformer +(29) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(29) ColumnarExchange +(32) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(33) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(33) ShuffledHashJoinExecTransformer +(37) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(41) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(40) ShuffledHashJoinExecTransformer +(45) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(47) Scan parquet +(53) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(48) FilterExecTransformer +(54) FilterExecTransformer Input [2]: [o_orderkey#X, o_orderstatus#X] Arguments: ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(49) ProjectExecTransformer +(55) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(50) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(51) ColumnarExchange +(57) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: X + +(58) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(53) InputAdapter +(60) InputAdapter Input [1]: [o_orderkey#X] -(54) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [o_orderkey#X] -(55) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(62) Scan parquet +(70) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(63) FilterExecTransformer +(71) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(64) ProjectExecTransformer +(72) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(65) WholeStageCodegenTransformer (X) +(73) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(66) ColumnarExchange +(74) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(75) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(76) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(68) InputAdapter +(77) InputAdapter Input [1]: [n_nationkey#X] -(69) InputIteratorTransformer +(78) InputIteratorTransformer Input [1]: [n_nationkey#X] -(70) ShuffledHashJoinExecTransformer +(79) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(72) FlushableHashAggregateExecTransformer +(81) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(73) ProjectExecTransformer +(82) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(74) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(75) ColumnarExchange +(84) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(85) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(77) InputAdapter +(86) InputAdapter Input [2]: [s_name#X, count#X] -(78) InputIteratorTransformer +(87) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(79) RegularHashAggregateExecTransformer +(88) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(80) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(81) VeloxColumnarToRowExec +(90) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(82) Scan parquet +(91) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(83) Filter +(92) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(84) Exchange +(93) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(94) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(86) Filter +(95) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(87) Project +(96) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(88) Exchange +(97) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) Scan parquet +(98) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(90) Exchange +(99) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) ShuffledHashJoin +(100) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(92) Scan parquet +(101) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(93) Filter +(102) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(94) Project +(103) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(95) Exchange +(104) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(105) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(97) Exchange +(106) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(99) Project +(108) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(100) Exchange +(109) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Scan parquet +(110) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(102) Filter +(111) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(103) Project +(112) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(104) Exchange +(113) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(106) Project +(115) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(107) Exchange +(116) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Scan parquet +(117) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(109) Filter +(118) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(110) Project +(119) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(111) Exchange +(120) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(121) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(113) Project +(122) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(114) HashAggregate +(123) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(115) Exchange +(124) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) HashAggregate +(125) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(117) TakeOrderedAndProject +(126) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(118) AdaptiveSparkPlan +(127) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt index add978887b0a..58bbd22fa13c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt @@ -1,43 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (46) +AdaptiveSparkPlan (49) +- == Final Plan == - VeloxColumnarToRowExec (33) - +- ^ SortExecTransformer (31) - +- ^ InputIteratorTransformer (30) - +- ShuffleQueryStage (28), Statistics(X) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (45) - +- Exchange (44) - +- HashAggregate (43) - +- Exchange (42) - +- HashAggregate (41) - +- Project (40) - +- ShuffledHashJoin LeftAnti BuildRight (39) - :- Exchange (36) - : +- Filter (35) - : +- Scan parquet (34) - +- Exchange (38) - +- Scan parquet (37) + Sort (48) + +- Exchange (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- ShuffledHashJoin LeftAnti BuildRight (42) + :- Exchange (39) + : +- Filter (38) + : +- Scan parquet (37) + +- Exchange (41) + +- Scan parquet (40) (1) Scan parquet @@ -59,318 +62,330 @@ Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) Scan parquet +(10) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [1]: [o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [1]: [o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(26) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) ColumnarExchange +(29) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(30) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(28) ShuffleQueryStage +(31) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(29) InputAdapter +(32) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) InputIteratorTransformer +(33) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(31) SortExecTransformer +(34) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(32) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(33) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) Scan parquet +(37) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(35) Filter +(38) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(36) Exchange +(39) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(40) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(40) Project +(43) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(41) HashAggregate +(44) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(42) Exchange +(45) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) HashAggregate +(46) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(44) Exchange +(47) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(48) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(46) AdaptiveSparkPlan +(49) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (65) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ FilterExecTransformer (48) - +- ^ Scan parquet (47) + VeloxColumnarToRowExec (61) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- ^ FlushableHashAggregateExecTransformer (53) + +- ^ ProjectExecTransformer (52) + +- ^ FilterExecTransformer (51) + +- ^ Scan parquet (50) +- == Initial Plan == - HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (67) + +- Exchange (66) + +- HashAggregate (65) + +- Project (64) + +- Filter (63) + +- Scan parquet (62) -(47) Scan parquet +(50) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(48) FilterExecTransformer +(51) FilterExecTransformer Input [2]: [c_phone#X, c_acctbal#X] Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(49) ProjectExecTransformer +(52) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(50) FlushableHashAggregateExecTransformer +(53) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(51) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(52) ColumnarExchange +(55) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(56) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(54) InputAdapter +(57) InputAdapter Input [2]: [sum#X, count#X] -(55) InputIteratorTransformer +(58) InputIteratorTransformer Input [2]: [sum#X, count#X] -(56) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(58) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(59) Scan parquet +(62) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(60) Filter +(63) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(61) Project +(64) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(62) HashAggregate +(65) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(63) Exchange +(66) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(67) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(65) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (65) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ FilterExecTransformer (48) - +- ^ Scan parquet (47) + VeloxColumnarToRowExec (61) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- ^ FlushableHashAggregateExecTransformer (53) + +- ^ ProjectExecTransformer (52) + +- ^ FilterExecTransformer (51) + +- ^ Scan parquet (50) +- == Initial Plan == - HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) \ No newline at end of file + HashAggregate (67) + +- Exchange (66) + +- HashAggregate (65) + +- Project (64) + +- Filter (63) + +- Scan parquet (62) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt index 69dc65c58e21..709a1700b5c7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt @@ -1,56 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (49) - : +- Project (48) - : +- ShuffledHashJoin Inner BuildLeft (47) - : :- Exchange (43) - : : +- Project (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Exchange (53) - +- Project (52) - +- Filter (51) - +- Scan parquet (50) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Project (59) + +- ShuffledHashJoin Inner BuildRight (58) + :- Exchange (53) + : +- Project (52) + : +- ShuffledHashJoin Inner BuildLeft (51) + : :- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Exchange (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -72,248 +76,264 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(23) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(42) Project +(46) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(43) Exchange +(47) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(48) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(45) Filter +(49) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(46) Exchange +(50) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(51) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(48) Project +(52) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Exchange +(53) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(54) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(51) Filter +(55) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(52) Project +(56) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(53) Exchange +(57) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(58) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(55) Project +(59) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(56) HashAggregate +(60) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(57) HashAggregate +(61) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt index d7f42dd7b351..ebf46078f68b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt @@ -1,47 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftSemi BuildRight (43) - :- Exchange (38) - : +- Project (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftSemi BuildRight (46) + :- Exchange (41) + : +- Project (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -63,202 +66,214 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [l_orderkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [l_orderkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(35) Scan parquet +(38) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(36) Filter +(39) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(37) Project +(40) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(38) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(41) Project +(44) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(42) Exchange +(45) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(44) Project +(47) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(45) HashAggregate +(48) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(46) Exchange +(49) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(50) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(48) Exchange +(51) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(52) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt index 129b02c9548e..8c3258e616f5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt @@ -1,115 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (134) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93), Statistics(X) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (133) - +- Exchange (132) - +- HashAggregate (131) - +- Exchange (130) - +- HashAggregate (129) - +- Project (128) - +- ShuffledHashJoin Inner BuildRight (127) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Project (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (126) - +- Project (125) - +- Filter (124) - +- Scan parquet (123) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Project (111) + : : : : +- Filter (110) + : : : : +- Scan parquet (109) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (137) + +- Project (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -131,562 +142,606 @@ Input [2]: [c_custkey#X, c_nationkey#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X + +(66) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: X + +(74) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [r_regionkey#X, r_name#X] Arguments: ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [1]: [r_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [1]: [r_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(99) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [2]: [n_name#X, revenue#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) Exchange +(108) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(109) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(99) Filter +(110) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(100) Project +(111) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(101) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(103) Project +(114) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(104) Exchange +(115) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(107) Exchange +(118) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(109) Project +(120) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(110) Exchange +(121) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(113) Exchange +(124) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(115) Project +(126) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(116) Exchange +(127) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(119) Exchange +(130) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(121) Project +(132) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(122) Exchange +(133) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(125) Project +(136) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(126) Exchange +(137) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(128) Project +(139) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(129) HashAggregate +(140) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(130) Exchange +(141) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) HashAggregate +(142) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(132) Exchange +(143) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(144) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(134) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/6.txt index 12d6c3ea85e4..415d85f46ee5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnot Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * l_discount#X) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt index 1e6af0683a39..fd60cdc1f478 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt @@ -1,110 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (138) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ SortExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ShuffleQueryStage (70), Statistics(X) - +- ReusedExchange (69) + VeloxColumnarToRowExec (100) + +- ^ SortExecTransformer (98) + +- ^ InputIteratorTransformer (97) + +- ShuffleQueryStage (95), Statistics(X) + +- ColumnarExchange (94) + +- VeloxAppendBatches (93) + +- ^ RegularHashAggregateExecTransformer (91) + +- ^ InputIteratorTransformer (90) + +- ShuffleQueryStage (88), Statistics(X) + +- ColumnarExchange (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ReusedExchange (78) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (111) - : : +- Project (110) - : : +- ShuffledHashJoin Inner BuildRight (109) - : : :- Exchange (105) - : : : +- Project (104) - : : : +- ShuffledHashJoin Inner BuildRight (103) - : : : :- Exchange (99) - : : : : +- Project (98) - : : : : +- ShuffledHashJoin Inner BuildLeft (97) - : : : : :- Exchange (93) - : : : : : +- Filter (92) - : : : : : +- Scan parquet (91) - : : : : +- Exchange (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Exchange (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (114) - : +- Filter (113) - : +- Scan parquet (112) - +- Exchange (120) - +- Filter (119) - +- Scan parquet (118) + Sort (137) + +- Exchange (136) + +- HashAggregate (135) + +- Exchange (134) + +- HashAggregate (133) + +- Project (132) + +- ShuffledHashJoin Inner BuildRight (131) + :- Exchange (127) + : +- Project (126) + : +- ShuffledHashJoin Inner BuildRight (125) + : :- Exchange (121) + : : +- Project (120) + : : +- ShuffledHashJoin Inner BuildRight (119) + : : :- Exchange (115) + : : : +- Project (114) + : : : +- ShuffledHashJoin Inner BuildRight (113) + : : : :- Exchange (109) + : : : : +- Project (108) + : : : : +- ShuffledHashJoin Inner BuildLeft (107) + : : : : :- Exchange (103) + : : : : : +- Filter (102) + : : : : : +- Scan parquet (101) + : : : : +- Exchange (106) + : : : : +- Filter (105) + : : : : +- Scan parquet (104) + : : : +- Exchange (112) + : : : +- Filter (111) + : : : +- Scan parquet (110) + : : +- Exchange (118) + : : +- Filter (117) + : : +- Scan parquet (116) + : +- Exchange (124) + : +- Filter (123) + : +- Scan parquet (122) + +- Exchange (130) + +- Filter (129) + +- Scan parquet (128) (1) Scan parquet @@ -126,534 +136,574 @@ Input [2]: [s_suppkey#X, s_nationkey#X] Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(23) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [o_orderkey#X, o_custkey#X] Arguments: (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: X + +(40) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [c_custkey#X, c_nationkey#X] Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(69) ReusedExchange [Reuses operator id: 58] +(78) ReusedExchange [Reuses operator id: 66] Output [2]: [n_nationkey#X, n_name#X] -(70) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(71) InputAdapter +(80) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(72) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(73) ShuffledHashJoinExecTransformer +(82) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(74) ProjectExecTransformer +(83) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(75) FlushableHashAggregateExecTransformer +(84) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) ProjectExecTransformer +(85) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) WholeStageCodegenTransformer (X) +(86) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(78) ColumnarExchange +(87) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(88) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(80) InputAdapter +(89) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(81) InputIteratorTransformer +(90) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(82) RegularHashAggregateExecTransformer +(91) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(83) WholeStageCodegenTransformer (X) +(92) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(84) ColumnarExchange +(93) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(94) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(95) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(86) InputAdapter +(96) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(87) InputIteratorTransformer +(97) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(88) SortExecTransformer +(98) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(89) WholeStageCodegenTransformer (X) +(99) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(90) VeloxColumnarToRowExec +(100) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(91) Scan parquet +(101) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(102) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(103) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(104) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(105) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Exchange +(106) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(98) Project +(108) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(102) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(104) Project +(114) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(105) Exchange +(115) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(116) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(117) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(118) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(110) Project +(120) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(111) Exchange +(121) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(122) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(113) Filter +(123) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(114) Exchange +(124) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(116) Project +(126) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(117) Exchange +(127) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(128) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(119) Filter +(129) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(120) Exchange +(130) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(122) Project +(132) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(123) HashAggregate +(133) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(124) Exchange +(134) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(135) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(126) Exchange +(136) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(137) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(128) AdaptiveSparkPlan +(138) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt index cfc03953fe18..5427956bb9d1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt @@ -1,150 +1,165 @@ == Physical Plan == -AdaptiveSparkPlan (177) +AdaptiveSparkPlan (192) +- == Final Plan == - VeloxColumnarToRowExec (125) - +- ^ SortExecTransformer (123) - +- ^ InputIteratorTransformer (122) - +- ShuffleQueryStage (120), Statistics(X) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ShuffleQueryStage (113), Statistics(X) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ShuffleQueryStage (96), Statistics(X) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ShuffleQueryStage (81), Statistics(X) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ShuffleQueryStage (66), Statistics(X) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ShuffleQueryStage (51), Statistics(X) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ShuffleQueryStage (36), Statistics(X) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ShuffleQueryStage (29), Statistics(X) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ FilterExecTransformer (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ShuffleQueryStage (44), Statistics(X) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ FilterExecTransformer (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ShuffleQueryStage (59), Statistics(X) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ FilterExecTransformer (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ShuffleQueryStage (74), Statistics(X) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ FilterExecTransformer (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ FilterExecTransformer (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ShuffleQueryStage (104), Statistics(X) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ FilterExecTransformer (100) - +- ^ Scan parquet (99) + VeloxColumnarToRowExec (140) + +- ^ SortExecTransformer (138) + +- ^ InputIteratorTransformer (137) + +- ShuffleQueryStage (135), Statistics(X) + +- ColumnarExchange (134) + +- VeloxAppendBatches (133) + +- ^ ProjectExecTransformer (131) + +- ^ RegularHashAggregateExecTransformer (130) + +- ^ InputIteratorTransformer (129) + +- ShuffleQueryStage (127), Statistics(X) + +- ColumnarExchange (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109), Statistics(X) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92), Statistics(X) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75), Statistics(X) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58), Statistics(X) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41), Statistics(X) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ FilterExecTransformer (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33), Statistics(X) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ FilterExecTransformer (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50), Statistics(X) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ FilterExecTransformer (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67), Statistics(X) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ FilterExecTransformer (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ FilterExecTransformer (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ FilterExecTransformer (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118), Statistics(X) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ FilterExecTransformer (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (176) - +- Exchange (175) - +- HashAggregate (174) - +- Exchange (173) - +- HashAggregate (172) - +- Project (171) - +- ShuffledHashJoin Inner BuildRight (170) - :- Exchange (165) - : +- Project (164) - : +- ShuffledHashJoin Inner BuildRight (163) - : :- Exchange (159) - : : +- Project (158) - : : +- ShuffledHashJoin Inner BuildRight (157) - : : :- Exchange (153) - : : : +- Project (152) - : : : +- ShuffledHashJoin Inner BuildRight (151) - : : : :- Exchange (147) - : : : : +- Project (146) - : : : : +- ShuffledHashJoin Inner BuildRight (145) - : : : : :- Exchange (141) - : : : : : +- Project (140) - : : : : : +- ShuffledHashJoin Inner BuildRight (139) - : : : : : :- Exchange (135) - : : : : : : +- Project (134) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) - : : : : : : :- Exchange (129) - : : : : : : : +- Project (128) - : : : : : : : +- Filter (127) - : : : : : : : +- Scan parquet (126) - : : : : : : +- Exchange (132) - : : : : : : +- Filter (131) - : : : : : : +- Scan parquet (130) - : : : : : +- Exchange (138) - : : : : : +- Filter (137) - : : : : : +- Scan parquet (136) - : : : : +- Exchange (144) - : : : : +- Filter (143) - : : : : +- Scan parquet (142) - : : : +- Exchange (150) - : : : +- Filter (149) - : : : +- Scan parquet (148) - : : +- Exchange (156) - : : +- Filter (155) - : : +- Scan parquet (154) - : +- Exchange (162) - : +- Filter (161) - : +- Scan parquet (160) - +- Exchange (169) - +- Project (168) - +- Filter (167) - +- Scan parquet (166) + Sort (191) + +- Exchange (190) + +- HashAggregate (189) + +- Exchange (188) + +- HashAggregate (187) + +- Project (186) + +- ShuffledHashJoin Inner BuildRight (185) + :- Exchange (180) + : +- Project (179) + : +- ShuffledHashJoin Inner BuildRight (178) + : :- Exchange (174) + : : +- Project (173) + : : +- ShuffledHashJoin Inner BuildRight (172) + : : :- Exchange (168) + : : : +- Project (167) + : : : +- ShuffledHashJoin Inner BuildRight (166) + : : : :- Exchange (162) + : : : : +- Project (161) + : : : : +- ShuffledHashJoin Inner BuildRight (160) + : : : : :- Exchange (156) + : : : : : +- Project (155) + : : : : : +- ShuffledHashJoin Inner BuildRight (154) + : : : : : :- Exchange (150) + : : : : : : +- Project (149) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (148) + : : : : : : :- Exchange (144) + : : : : : : : +- Project (143) + : : : : : : : +- Filter (142) + : : : : : : : +- Scan parquet (141) + : : : : : : +- Exchange (147) + : : : : : : +- Filter (146) + : : : : : : +- Scan parquet (145) + : : : : : +- Exchange (153) + : : : : : +- Filter (152) + : : : : : +- Scan parquet (151) + : : : : +- Exchange (159) + : : : : +- Filter (158) + : : : : +- Scan parquet (157) + : : : +- Exchange (165) + : : : +- Filter (164) + : : : +- Scan parquet (163) + : : +- Exchange (171) + : : +- Filter (170) + : : +- Scan parquet (169) + : +- Exchange (177) + : +- Filter (176) + : +- Scan parquet (175) + +- Exchange (184) + +- Project (183) + +- Filter (182) + +- Scan parquet (181) (1) Scan parquet @@ -166,746 +181,806 @@ Input [2]: [p_partkey#X, p_type#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [c_custkey#X, c_nationkey#X] Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(79) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(80) ColumnarExchange +(90) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: X + +(91) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(92) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(82) InputAdapter +(93) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(83) InputIteratorTransformer +(94) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(84) Scan parquet +(95) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) FilterExecTransformer +(96) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(86) ProjectExecTransformer +(97) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(92) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(93) ProjectExecTransformer +(105) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(94) WholeStageCodegenTransformer (X) +(106) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(95) ColumnarExchange +(107) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: X + +(108) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(109) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(97) InputAdapter +(110) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(98) InputIteratorTransformer +(111) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(99) Scan parquet +(112) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(100) FilterExecTransformer +(113) FilterExecTransformer Input [2]: [r_regionkey#X, r_name#X] Arguments: ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(101) ProjectExecTransformer +(114) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(102) WholeStageCodegenTransformer (X) +(115) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(103) ColumnarExchange +(116) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(117) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(104) ShuffleQueryStage +(118) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(105) InputAdapter +(119) InputAdapter Input [1]: [r_regionkey#X] -(106) InputIteratorTransformer +(120) InputIteratorTransformer Input [1]: [r_regionkey#X] -(107) ShuffledHashJoinExecTransformer +(121) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(108) ProjectExecTransformer +(122) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(109) FlushableHashAggregateExecTransformer +(123) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(110) ProjectExecTransformer +(124) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(111) WholeStageCodegenTransformer (X) +(125) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(112) ColumnarExchange +(126) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(113) ShuffleQueryStage +(127) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(114) InputAdapter +(128) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(115) InputIteratorTransformer +(129) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(116) RegularHashAggregateExecTransformer +(130) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(117) ProjectExecTransformer +(131) ProjectExecTransformer Output [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(118) WholeStageCodegenTransformer (X) +(132) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(119) ColumnarExchange +(133) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(134) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(120) ShuffleQueryStage +(135) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(121) InputAdapter +(136) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(122) InputIteratorTransformer +(137) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(123) SortExecTransformer +(138) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(124) WholeStageCodegenTransformer (X) +(139) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(125) VeloxColumnarToRowExec +(140) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(126) Scan parquet +(141) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(127) Filter +(142) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(128) Project +(143) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(129) Exchange +(144) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) Scan parquet +(145) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(131) Filter +(146) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(132) Exchange +(147) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) ShuffledHashJoin +(148) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(134) Project +(149) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(135) Exchange +(150) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Scan parquet +(151) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(137) Filter +(152) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(138) Exchange +(153) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(154) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(140) Project +(155) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(141) Exchange +(156) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) Scan parquet +(157) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(143) Filter +(158) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(144) Exchange +(159) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) ShuffledHashJoin +(160) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(146) Project +(161) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(147) Exchange +(162) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Scan parquet +(163) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(149) Filter +(164) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(150) Exchange +(165) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) ShuffledHashJoin +(166) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(152) Project +(167) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(153) Exchange +(168) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) Scan parquet +(169) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(155) Filter +(170) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(156) Exchange +(171) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) ShuffledHashJoin +(172) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(158) Project +(173) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(159) Exchange +(174) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Scan parquet +(175) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(161) Filter +(176) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(162) Exchange +(177) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) ShuffledHashJoin +(178) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(164) Project +(179) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(165) Exchange +(180) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) Scan parquet +(181) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(167) Filter +(182) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(168) Project +(183) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(169) Exchange +(184) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) ShuffledHashJoin +(185) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(171) Project +(186) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(172) HashAggregate +(187) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(173) Exchange +(188) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(174) HashAggregate +(189) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(175) Exchange +(190) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(191) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(177) AdaptiveSparkPlan +(192) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt index 9a8c9a87aac2..af275ac9926c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt @@ -1,114 +1,125 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (144) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93), Statistics(X) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- HashAggregate (130) - +- Exchange (129) - +- HashAggregate (128) - +- Project (127) - +- ShuffledHashJoin Inner BuildRight (126) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (98) - : : : : : +- Project (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (125) - +- Filter (124) - +- Scan parquet (123) + Sort (143) + +- Exchange (142) + +- HashAggregate (141) + +- Exchange (140) + +- HashAggregate (139) + +- Project (138) + +- ShuffledHashJoin Inner BuildRight (137) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (109) + : : : : : +- Project (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -130,558 +141,602 @@ Input [2]: [p_partkey#X, p_name#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: X + +(57) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: isnotnull(o_orderkey#X) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: X + +(74) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(99) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(97) Project +(108) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(109) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(110) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(100) Filter +(111) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(101) Exchange +(112) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(103) Project +(114) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(115) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(118) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(109) Project +(120) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(121) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(113) Exchange +(124) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(115) Project +(126) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(116) Exchange +(127) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(119) Exchange +(130) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(121) Project +(132) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(122) Exchange +(133) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(125) Exchange +(136) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(137) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(127) Project +(138) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(128) HashAggregate +(139) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(129) Exchange +(140) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) HashAggregate +(141) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(131) Exchange +(142) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(143) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(133) AdaptiveSparkPlan +(144) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file From 9e020173a01a7d90fdba2c25b597cd45f33e1e61 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Fri, 14 Jun 2024 16:13:13 +0800 Subject: [PATCH 5/9] fixup --- .../FlushableHashAggregateRule.scala | 115 ++--- .../v1-bhj-ras/spark32/1.txt | 99 ++-- .../v1-bhj-ras/spark32/10.txt | 4 +- .../v1-bhj-ras/spark32/11.txt | 175 +++---- .../v1-bhj-ras/spark32/12.txt | 133 ++--- .../v1-bhj-ras/spark32/13.txt | 176 +++---- .../v1-bhj-ras/spark32/14.txt | 4 +- .../v1-bhj-ras/spark32/15.txt | 135 +++--- .../v1-bhj-ras/spark32/16.txt | 192 ++++---- .../v1-bhj-ras/spark32/17.txt | 4 +- .../v1-bhj-ras/spark32/18.txt | 8 +- .../v1-bhj-ras/spark32/19.txt | 4 +- .../v1-bhj-ras/spark32/20.txt | 295 +++++------ .../v1-bhj-ras/spark32/21.txt | 4 +- .../v1-bhj-ras/spark32/22.txt | 121 ++--- .../v1-bhj-ras/spark32/3.txt | 4 +- .../v1-bhj-ras/spark32/4.txt | 139 +++--- .../v1-bhj-ras/spark32/5.txt | 275 +++++------ .../v1-bhj-ras/spark32/6.txt | 4 +- .../v1-bhj-ras/spark32/7.txt | 259 +++++----- .../v1-bhj-ras/spark32/8.txt | 347 ++++++------- .../v1-bhj-ras/spark32/9.txt | 269 ++++++----- .../v1-bhj-ras/spark33/1.txt | 99 ++-- .../v1-bhj-ras/spark33/10.txt | 4 +- .../v1-bhj-ras/spark33/11.txt | 340 ++++++------- .../v1-bhj-ras/spark33/12.txt | 133 ++--- .../v1-bhj-ras/spark33/13.txt | 176 +++---- .../v1-bhj-ras/spark33/14.txt | 4 +- .../v1-bhj-ras/spark33/15.txt | 228 ++++----- .../v1-bhj-ras/spark33/16.txt | 192 ++++---- .../v1-bhj-ras/spark33/17.txt | 4 +- .../v1-bhj-ras/spark33/18.txt | 8 +- .../v1-bhj-ras/spark33/19.txt | 4 +- .../v1-bhj-ras/spark33/20.txt | 287 +++++------ .../v1-bhj-ras/spark33/21.txt | 4 +- .../v1-bhj-ras/spark33/22.txt | 196 ++++---- .../v1-bhj-ras/spark33/3.txt | 4 +- .../v1-bhj-ras/spark33/4.txt | 139 +++--- .../v1-bhj-ras/spark33/5.txt | 275 +++++------ .../v1-bhj-ras/spark33/6.txt | 4 +- .../v1-bhj-ras/spark33/7.txt | 259 +++++----- .../v1-bhj-ras/spark33/8.txt | 347 ++++++------- .../v1-bhj-ras/spark33/9.txt | 269 ++++++----- .../v1-bhj-ras/spark34/1.txt | 99 ++-- .../v1-bhj-ras/spark34/10.txt | 4 +- .../v1-bhj-ras/spark34/11.txt | 340 ++++++------- .../v1-bhj-ras/spark34/12.txt | 133 ++--- .../v1-bhj-ras/spark34/13.txt | 176 +++---- .../v1-bhj-ras/spark34/14.txt | 4 +- .../v1-bhj-ras/spark34/15.txt | 228 ++++----- .../v1-bhj-ras/spark34/16.txt | 192 ++++---- .../v1-bhj-ras/spark34/17.txt | 4 +- .../v1-bhj-ras/spark34/18.txt | 8 +- .../v1-bhj-ras/spark34/19.txt | 4 +- .../v1-bhj-ras/spark34/20.txt | 283 +++++------ .../v1-bhj-ras/spark34/21.txt | 4 +- .../v1-bhj-ras/spark34/22.txt | 196 ++++---- .../v1-bhj-ras/spark34/3.txt | 4 +- .../v1-bhj-ras/spark34/4.txt | 139 +++--- .../v1-bhj-ras/spark34/5.txt | 275 +++++------ .../v1-bhj-ras/spark34/6.txt | 4 +- .../v1-bhj-ras/spark34/7.txt | 259 +++++----- .../v1-bhj-ras/spark34/8.txt | 347 ++++++------- .../v1-bhj-ras/spark34/9.txt | 269 ++++++----- .../tpch-approved-plan/v1-bhj/spark32/1.txt | 99 ++-- .../tpch-approved-plan/v1-bhj/spark32/10.txt | 4 +- .../tpch-approved-plan/v1-bhj/spark32/11.txt | 175 +++---- .../tpch-approved-plan/v1-bhj/spark32/12.txt | 133 ++--- .../tpch-approved-plan/v1-bhj/spark32/13.txt | 176 +++---- .../tpch-approved-plan/v1-bhj/spark32/14.txt | 4 +- .../tpch-approved-plan/v1-bhj/spark32/15.txt | 135 +++--- .../tpch-approved-plan/v1-bhj/spark32/16.txt | 192 ++++---- .../tpch-approved-plan/v1-bhj/spark32/17.txt | 4 +- .../tpch-approved-plan/v1-bhj/spark32/18.txt | 8 +- .../tpch-approved-plan/v1-bhj/spark32/19.txt | 4 +- .../tpch-approved-plan/v1-bhj/spark32/20.txt | 295 +++++------ .../tpch-approved-plan/v1-bhj/spark32/21.txt | 4 +- .../tpch-approved-plan/v1-bhj/spark32/22.txt | 121 ++--- .../tpch-approved-plan/v1-bhj/spark32/3.txt | 4 +- .../tpch-approved-plan/v1-bhj/spark32/4.txt | 139 +++--- .../tpch-approved-plan/v1-bhj/spark32/5.txt | 275 +++++------ .../tpch-approved-plan/v1-bhj/spark32/6.txt | 4 +- .../tpch-approved-plan/v1-bhj/spark32/7.txt | 259 +++++----- .../tpch-approved-plan/v1-bhj/spark32/8.txt | 347 ++++++------- .../tpch-approved-plan/v1-bhj/spark32/9.txt | 269 ++++++----- .../tpch-approved-plan/v1-bhj/spark33/1.txt | 99 ++-- .../tpch-approved-plan/v1-bhj/spark33/10.txt | 4 +- .../tpch-approved-plan/v1-bhj/spark33/11.txt | 340 ++++++------- .../tpch-approved-plan/v1-bhj/spark33/12.txt | 133 ++--- .../tpch-approved-plan/v1-bhj/spark33/13.txt | 176 +++---- .../tpch-approved-plan/v1-bhj/spark33/14.txt | 4 +- .../tpch-approved-plan/v1-bhj/spark33/15.txt | 228 ++++----- .../tpch-approved-plan/v1-bhj/spark33/16.txt | 192 ++++---- .../tpch-approved-plan/v1-bhj/spark33/17.txt | 4 +- .../tpch-approved-plan/v1-bhj/spark33/18.txt | 8 +- .../tpch-approved-plan/v1-bhj/spark33/19.txt | 4 +- .../tpch-approved-plan/v1-bhj/spark33/20.txt | 287 +++++------ .../tpch-approved-plan/v1-bhj/spark33/21.txt | 4 +- .../tpch-approved-plan/v1-bhj/spark33/22.txt | 229 ++++----- .../tpch-approved-plan/v1-bhj/spark33/3.txt | 4 +- .../tpch-approved-plan/v1-bhj/spark33/4.txt | 139 +++--- .../tpch-approved-plan/v1-bhj/spark33/5.txt | 275 +++++------ .../tpch-approved-plan/v1-bhj/spark33/6.txt | 4 +- .../tpch-approved-plan/v1-bhj/spark33/7.txt | 259 +++++----- .../tpch-approved-plan/v1-bhj/spark33/8.txt | 347 ++++++------- .../tpch-approved-plan/v1-bhj/spark33/9.txt | 269 ++++++----- .../tpch-approved-plan/v1-bhj/spark34/1.txt | 99 ++-- .../tpch-approved-plan/v1-bhj/spark34/10.txt | 4 +- .../tpch-approved-plan/v1-bhj/spark34/11.txt | 340 ++++++------- .../tpch-approved-plan/v1-bhj/spark34/12.txt | 133 ++--- .../tpch-approved-plan/v1-bhj/spark34/13.txt | 176 +++---- .../tpch-approved-plan/v1-bhj/spark34/14.txt | 4 +- .../tpch-approved-plan/v1-bhj/spark34/15.txt | 228 ++++----- .../tpch-approved-plan/v1-bhj/spark34/16.txt | 192 ++++---- .../tpch-approved-plan/v1-bhj/spark34/17.txt | 4 +- .../tpch-approved-plan/v1-bhj/spark34/18.txt | 8 +- .../tpch-approved-plan/v1-bhj/spark34/19.txt | 4 +- .../tpch-approved-plan/v1-bhj/spark34/20.txt | 283 +++++------ .../tpch-approved-plan/v1-bhj/spark34/21.txt | 4 +- .../tpch-approved-plan/v1-bhj/spark34/22.txt | 229 ++++----- .../tpch-approved-plan/v1-bhj/spark34/3.txt | 4 +- .../tpch-approved-plan/v1-bhj/spark34/4.txt | 139 +++--- .../tpch-approved-plan/v1-bhj/spark34/5.txt | 275 +++++------ .../tpch-approved-plan/v1-bhj/spark34/6.txt | 4 +- .../tpch-approved-plan/v1-bhj/spark34/7.txt | 259 +++++----- .../tpch-approved-plan/v1-bhj/spark34/8.txt | 347 ++++++------- .../tpch-approved-plan/v1-bhj/spark34/9.txt | 269 ++++++----- .../tpch-approved-plan/v1-ras/spark32/1.txt | 99 ++-- .../tpch-approved-plan/v1-ras/spark32/10.txt | 233 ++++----- .../tpch-approved-plan/v1-ras/spark32/11.txt | 207 ++++---- .../tpch-approved-plan/v1-ras/spark32/12.txt | 149 +++--- .../tpch-approved-plan/v1-ras/spark32/13.txt | 157 +++--- .../tpch-approved-plan/v1-ras/spark32/15.txt | 135 +++--- .../tpch-approved-plan/v1-ras/spark32/16.txt | 210 ++++---- .../tpch-approved-plan/v1-ras/spark32/17.txt | 145 +++--- .../tpch-approved-plan/v1-ras/spark32/18.txt | 313 ++++++------ .../tpch-approved-plan/v1-ras/spark32/21.txt | 309 ++++++------ .../tpch-approved-plan/v1-ras/spark32/22.txt | 139 +++--- .../tpch-approved-plan/v1-ras/spark32/4.txt | 153 +++--- .../tpch-approved-plan/v1-ras/spark32/5.txt | 353 +++++++------- .../tpch-approved-plan/v1-ras/spark32/6.txt | 4 +- .../tpch-approved-plan/v1-ras/spark32/7.txt | 337 ++++++------- .../tpch-approved-plan/v1-ras/spark32/8.txt | 457 +++++++++--------- .../tpch-approved-plan/v1-ras/spark32/9.txt | 349 ++++++------- .../tpch-approved-plan/v1-ras/spark33/1.txt | 99 ++-- .../tpch-approved-plan/v1-ras/spark33/10.txt | 233 ++++----- .../tpch-approved-plan/v1-ras/spark33/11.txt | 399 +++++++-------- .../tpch-approved-plan/v1-ras/spark33/12.txt | 149 +++--- .../tpch-approved-plan/v1-ras/spark33/13.txt | 157 +++--- .../tpch-approved-plan/v1-ras/spark33/15.txt | 228 ++++----- .../tpch-approved-plan/v1-ras/spark33/16.txt | 210 ++++---- .../tpch-approved-plan/v1-ras/spark33/17.txt | 145 +++--- .../tpch-approved-plan/v1-ras/spark33/18.txt | 313 ++++++------ .../tpch-approved-plan/v1-ras/spark33/21.txt | 305 ++++++------ .../tpch-approved-plan/v1-ras/spark33/22.txt | 214 ++++---- .../tpch-approved-plan/v1-ras/spark33/4.txt | 153 +++--- .../tpch-approved-plan/v1-ras/spark33/5.txt | 353 +++++++------- .../tpch-approved-plan/v1-ras/spark33/6.txt | 4 +- .../tpch-approved-plan/v1-ras/spark33/7.txt | 337 ++++++------- .../tpch-approved-plan/v1-ras/spark33/8.txt | 457 +++++++++--------- .../tpch-approved-plan/v1-ras/spark33/9.txt | 349 ++++++------- .../tpch-approved-plan/v1-ras/spark34/1.txt | 99 ++-- .../tpch-approved-plan/v1-ras/spark34/10.txt | 233 ++++----- .../tpch-approved-plan/v1-ras/spark34/11.txt | 399 +++++++-------- .../tpch-approved-plan/v1-ras/spark34/12.txt | 149 +++--- .../tpch-approved-plan/v1-ras/spark34/13.txt | 157 +++--- .../tpch-approved-plan/v1-ras/spark34/15.txt | 228 ++++----- .../tpch-approved-plan/v1-ras/spark34/16.txt | 210 ++++---- .../tpch-approved-plan/v1-ras/spark34/17.txt | 145 +++--- .../tpch-approved-plan/v1-ras/spark34/18.txt | 313 ++++++------ .../tpch-approved-plan/v1-ras/spark34/21.txt | 305 ++++++------ .../tpch-approved-plan/v1-ras/spark34/22.txt | 214 ++++---- .../tpch-approved-plan/v1-ras/spark34/4.txt | 153 +++--- .../tpch-approved-plan/v1-ras/spark34/5.txt | 353 +++++++------- .../tpch-approved-plan/v1-ras/spark34/6.txt | 4 +- .../tpch-approved-plan/v1-ras/spark34/7.txt | 337 ++++++------- .../tpch-approved-plan/v1-ras/spark34/8.txt | 457 +++++++++--------- .../tpch-approved-plan/v1-ras/spark34/9.txt | 349 ++++++------- .../tpch-approved-plan/v1/spark32/1.txt | 99 ++-- .../tpch-approved-plan/v1/spark32/10.txt | 233 ++++----- .../tpch-approved-plan/v1/spark32/11.txt | 207 ++++---- .../tpch-approved-plan/v1/spark32/12.txt | 149 +++--- .../tpch-approved-plan/v1/spark32/13.txt | 157 +++--- .../tpch-approved-plan/v1/spark32/15.txt | 135 +++--- .../tpch-approved-plan/v1/spark32/16.txt | 210 ++++---- .../tpch-approved-plan/v1/spark32/17.txt | 145 +++--- .../tpch-approved-plan/v1/spark32/18.txt | 313 ++++++------ .../tpch-approved-plan/v1/spark32/21.txt | 309 ++++++------ .../tpch-approved-plan/v1/spark32/22.txt | 139 +++--- .../tpch-approved-plan/v1/spark32/4.txt | 153 +++--- .../tpch-approved-plan/v1/spark32/5.txt | 353 +++++++------- .../tpch-approved-plan/v1/spark32/6.txt | 4 +- .../tpch-approved-plan/v1/spark32/7.txt | 337 ++++++------- .../tpch-approved-plan/v1/spark32/8.txt | 457 +++++++++--------- .../tpch-approved-plan/v1/spark32/9.txt | 349 ++++++------- .../tpch-approved-plan/v1/spark33/1.txt | 99 ++-- .../tpch-approved-plan/v1/spark33/10.txt | 233 ++++----- .../tpch-approved-plan/v1/spark33/11.txt | 399 +++++++-------- .../tpch-approved-plan/v1/spark33/12.txt | 149 +++--- .../tpch-approved-plan/v1/spark33/13.txt | 157 +++--- .../tpch-approved-plan/v1/spark33/15.txt | 228 ++++----- .../tpch-approved-plan/v1/spark33/16.txt | 210 ++++---- .../tpch-approved-plan/v1/spark33/17.txt | 145 +++--- .../tpch-approved-plan/v1/spark33/18.txt | 313 ++++++------ .../tpch-approved-plan/v1/spark33/21.txt | 305 ++++++------ .../tpch-approved-plan/v1/spark33/22.txt | 247 +++++----- .../tpch-approved-plan/v1/spark33/4.txt | 153 +++--- .../tpch-approved-plan/v1/spark33/5.txt | 353 +++++++------- .../tpch-approved-plan/v1/spark33/6.txt | 4 +- .../tpch-approved-plan/v1/spark33/7.txt | 337 ++++++------- .../tpch-approved-plan/v1/spark33/8.txt | 457 +++++++++--------- .../tpch-approved-plan/v1/spark33/9.txt | 349 ++++++------- .../tpch-approved-plan/v1/spark34/1.txt | 99 ++-- .../tpch-approved-plan/v1/spark34/10.txt | 233 ++++----- .../tpch-approved-plan/v1/spark34/11.txt | 399 +++++++-------- .../tpch-approved-plan/v1/spark34/12.txt | 149 +++--- .../tpch-approved-plan/v1/spark34/13.txt | 157 +++--- .../tpch-approved-plan/v1/spark34/15.txt | 228 ++++----- .../tpch-approved-plan/v1/spark34/16.txt | 210 ++++---- .../tpch-approved-plan/v1/spark34/17.txt | 145 +++--- .../tpch-approved-plan/v1/spark34/18.txt | 313 ++++++------ .../tpch-approved-plan/v1/spark34/21.txt | 305 ++++++------ .../tpch-approved-plan/v1/spark34/22.txt | 247 +++++----- .../tpch-approved-plan/v1/spark34/4.txt | 153 +++--- .../tpch-approved-plan/v1/spark34/5.txt | 353 +++++++------- .../tpch-approved-plan/v1/spark34/6.txt | 4 +- .../tpch-approved-plan/v1/spark34/7.txt | 337 ++++++------- .../tpch-approved-plan/v1/spark34/8.txt | 457 +++++++++--------- .../tpch-approved-plan/v1/spark34/9.txt | 349 ++++++------- 229 files changed, 21199 insertions(+), 20164 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala b/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala index c4b9a768fa84..3910acbebb4c 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala @@ -32,73 +32,56 @@ import org.apache.spark.sql.execution.exchange.ShuffleExchangeLike case class FlushableHashAggregateRule(session: SparkSession) extends Rule[SparkPlan] { import FlushableHashAggregateRule._ override def apply(plan: SparkPlan): SparkPlan = plan.transformUp { - case ShuffleAndChild(shuffle, child) => + case s: ShuffleExchangeLike => // If an exchange follows a hash aggregate in which all functions are in partial mode, // then it's safe to convert the hash aggregate to flushable hash aggregate. - child match { - case HashAggPropagatedToShuffle(proj, agg) => - shuffle.withNewChildren( - Seq(proj.withNewChildren(Seq(FlushableHashAggregateExecTransformer( - agg.requiredChildDistributionExpressions, - agg.groupingExpressions, - agg.aggregateExpressions, - agg.aggregateAttributes, - agg.initialInputBufferOffset, - agg.resultExpressions, - agg.child - ))))) - case HashAggWithShuffle(agg) => - shuffle.withNewChildren( - Seq(FlushableHashAggregateExecTransformer( - agg.requiredChildDistributionExpressions, - agg.groupingExpressions, - agg.aggregateExpressions, - agg.aggregateAttributes, - agg.initialInputBufferOffset, - agg.resultExpressions, - agg.child - ))) - case _ => - shuffle - } + val out = s.withNewChildren( + List( + replaceEligibleAggregates(s.child) { + agg => + FlushableHashAggregateExecTransformer( + agg.requiredChildDistributionExpressions, + agg.groupingExpressions, + agg.aggregateExpressions, + agg.aggregateAttributes, + agg.initialInputBufferOffset, + agg.resultExpressions, + agg.child + ) + } + ) + ) + out } -} -object HashAggPropagatedToShuffle { - def unapply( - plan: SparkPlan): Option[(ProjectExecTransformer, RegularHashAggregateExecTransformer)] = { - if (!plan.isInstanceOf[ProjectExecTransformer]) { - return None - } - val proj = plan.asInstanceOf[ProjectExecTransformer] - val child = proj.child - if (!child.isInstanceOf[RegularHashAggregateExecTransformer]) { - return None - } - val agg = child.asInstanceOf[RegularHashAggregateExecTransformer] - if (!agg.aggregateExpressions.forall(p => p.mode == Partial || p.mode == PartialMerge)) { - return None + private def replaceEligibleAggregates(plan: SparkPlan)( + func: RegularHashAggregateExecTransformer => SparkPlan): SparkPlan = { + def transformDown: SparkPlan => SparkPlan = { + case agg: RegularHashAggregateExecTransformer + if !agg.aggregateExpressions.forall(p => p.mode == Partial || p.mode == PartialMerge) => + // Not a intermediate agg. Skip. + agg + case agg: RegularHashAggregateExecTransformer + if isAggInputAlreadyDistributedWithAggKeys(agg) => + // Data already grouped by aggregate keys, Skip. + agg + case agg: RegularHashAggregateExecTransformer => + func(agg) + case p if !canPropagate(p) => p + case other => other.withNewChildren(other.children.map(transformDown)) } - if (FlushableHashAggregateRule.isAggInputAlreadyDistributedWithAggKeys(agg)) { - return None + + val out = transformDown(plan) + if (out.fastEquals(plan)) { + return plan } - Some((proj, agg)) + out } -} -object HashAggWithShuffle { - def unapply(plan: SparkPlan): Option[RegularHashAggregateExecTransformer] = { - if (!plan.isInstanceOf[RegularHashAggregateExecTransformer]) { - return None - } - val agg = plan.asInstanceOf[RegularHashAggregateExecTransformer] - if (!agg.aggregateExpressions.forall(p => p.mode == Partial || p.mode == PartialMerge)) { - return None - } - if (FlushableHashAggregateRule.isAggInputAlreadyDistributedWithAggKeys(agg)) { - return None - } - Some(agg) + private def canPropagate(plan: SparkPlan): Boolean = plan match { + case _: ProjectExecTransformer => true + case _: VeloxAppendBatchesExec => true + case _ => false } } @@ -113,7 +96,7 @@ object FlushableHashAggregateRule { * only on a single partition among the whole cluster. Spark's planner may use this information to * perform optimizations like doing "partial_count(a, b, c)" directly on the output data. */ - def isAggInputAlreadyDistributedWithAggKeys(agg: HashAggregateExecTransformer): Boolean = { + private def isAggInputAlreadyDistributedWithAggKeys(agg: HashAggregateExecTransformer): Boolean = { if (agg.groupingExpressions.isEmpty) { // Empty grouping set () should not be satisfied by any partitioning patterns. // E.g., @@ -126,16 +109,4 @@ object FlushableHashAggregateRule { val distribution = ClusteredDistribution(agg.groupingExpressions) agg.child.outputPartitioning.satisfies(distribution) } - - private object ShuffleAndChild { - def unapply(plan: SparkPlan): Option[(SparkPlan, SparkPlan)] = plan match { - case s: ShuffleExchangeLike => - val child = s.child match { - case VeloxAppendBatchesExec(child, _) => child - case other => other - } - Some(s, child) - case other => None - } - } } diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt index 69a930ea8545..53edb933c1fb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (29) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (28) - +- Exchange (27) - +- HashAggregate (26) - +- Exchange (25) - +- HashAggregate (24) - +- Project (23) - +- Filter (22) - +- Scan parquet (21) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -57,101 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) VeloxAppendBatches +(14) VeloxAppendBatches Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) ColumnarExchange +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(18) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(20) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(21) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(22) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(23) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(24) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(25) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(27) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(29) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt index 8f93964d76f5..29f28ac26a6b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt @@ -10,7 +10,7 @@ AdaptiveSparkPlan (68) +- ColumnarExchange (36) +- VeloxAppendBatches (35) +- ^ ProjectExecTransformer (33) - +- ^ RegularHashAggregateExecTransformer (32) + +- ^ FlushableHashAggregateExecTransformer (32) +- ^ ProjectExecTransformer (31) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) :- ^ ProjectExecTransformer (22) @@ -195,7 +195,7 @@ Join condition: None Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(32) RegularHashAggregateExecTransformer +(32) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt index 0e98c4c0edd7..b8f466eb9456 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt @@ -1,56 +1,57 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- ^ SortExecTransformer (37) - +- ^ InputIteratorTransformer (36) - +- ShuffleQueryStage (34) - +- ColumnarExchange (33) - +- VeloxAppendBatches (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7) - : +- ColumnarBroadcastExchange (6) - : +- ^ NoopFilter (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ NoopFilter (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ FilterExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7) + : +- ColumnarBroadcastExchange (6) + : +- ^ NoopFilter (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ NoopFilter (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (58) - +- Exchange (57) - +- Filter (56) - +- HashAggregate (55) - +- Exchange (54) - +- HashAggregate (53) - +- Project (52) - +- BroadcastHashJoin Inner BuildRight (51) - :- Project (46) - : +- BroadcastHashJoin Inner BuildRight (45) - : :- Filter (41) - : : +- Scan parquet (40) - : +- BroadcastExchange (44) - : +- Filter (43) - : +- Scan parquet (42) - +- BroadcastExchange (50) - +- Project (49) - +- Filter (48) - +- Scan parquet (47) + Sort (59) + +- Exchange (58) + +- Filter (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- BroadcastHashJoin Inner BuildRight (52) + :- Project (47) + : +- BroadcastHashJoin Inner BuildRight (46) + : :- Filter (42) + : : +- Scan parquet (41) + : +- BroadcastExchange (45) + : +- Filter (44) + : +- Scan parquet (43) + +- BroadcastExchange (51) + +- Project (50) + +- Filter (49) + +- Scan parquet (48) (1) Scan parquet @@ -159,157 +160,161 @@ Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(25) ColumnarExchange +(25) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(26) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(27) InputAdapter +(28) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(28) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(29) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(30) FilterExecTransformer +(31) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) VeloxAppendBatches +(33) VeloxAppendBatches Input [2]: [ps_partkey#X, value#X] Arguments: X -(33) ColumnarExchange +(34) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(34) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(35) InputAdapter +(36) InputAdapter Input [2]: [ps_partkey#X, value#X] -(36) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(37) SortExecTransformer +(38) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(38) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(39) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(40) Scan parquet +(41) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(41) Filter +(42) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(42) Scan parquet +(43) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(43) Filter +(44) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(44) BroadcastExchange +(45) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(45) BroadcastHashJoin +(46) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(46) Project +(47) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(47) Scan parquet +(48) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(48) Filter +(49) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(49) Project +(50) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(50) BroadcastExchange +(51) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(51) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(52) Project +(53) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(53) HashAggregate +(54) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(54) Exchange +(55) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) HashAggregate +(56) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(56) Filter +(57) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(57) Exchange +(58) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) Sort +(59) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(59) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt index d4942783d0ff..1e60a93910a7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt @@ -1,42 +1,43 @@ == Physical Plan == -AdaptiveSparkPlan (43) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- VeloxAppendBatches (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (42) - +- Exchange (41) - +- HashAggregate (40) - +- Exchange (39) - +- HashAggregate (38) - +- Project (37) - +- BroadcastHashJoin Inner BuildLeft (36) - :- BroadcastExchange (32) - : +- Filter (31) - : +- Scan parquet (30) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (43) + +- Exchange (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- BroadcastHashJoin Inner BuildLeft (37) + :- BroadcastExchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -107,125 +108,129 @@ Input [3]: [l_shipmode#X, sum#X, sum#X] Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(16) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(20) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) VeloxAppendBatches +(23) VeloxAppendBatches Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(23) ColumnarExchange +(24) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(27) SortExecTransformer +(28) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(30) Scan parquet +(31) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(32) BroadcastExchange +(33) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(33) Scan parquet +(34) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(34) Filter +(35) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(35) Project +(36) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(36) BroadcastHashJoin +(37) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(37) Project +(38) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(38) HashAggregate +(39) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(39) Exchange +(40) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) HashAggregate +(41) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(41) Exchange +(42) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Sort +(43) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(43) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt index 4320ccecc911..c525944e1262 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt @@ -1,50 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32) - +- ColumnarExchange (31) - +- VeloxAppendBatches (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ NoopFilter (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ NoopFilter (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (52) - +- Exchange (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- BroadcastHashJoin LeftOuter BuildRight (43) - :- Scan parquet (38) - +- BroadcastExchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- BroadcastHashJoin LeftOuter BuildRight (45) + :- Scan parquet (40) + +- BroadcastExchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -110,178 +112,186 @@ Input [2]: [c_custkey#X, count#X] Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, count#X] +Arguments: X + +(16) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [c_custkey#X, count#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [c_custkey#X, count#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(22) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) VeloxAppendBatches +(32) VeloxAppendBatches Input [2]: [c_count#X, custdist#X] Arguments: X -(31) ColumnarExchange +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(33) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(34) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(35) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(36) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(37) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(38) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(39) Scan parquet +(41) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(42) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(43) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) BroadcastExchange +(44) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(43) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(46) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) Exchange +(48) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(48) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(49) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(51) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(53) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt index b79d92ba4257..fbfcf07030eb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt @@ -8,7 +8,7 @@ AdaptiveSparkPlan (35) +- ShuffleQueryStage (17) +- ColumnarExchange (16) +- VeloxAppendBatches (15) - +- ^ RegularHashAggregateExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) @@ -86,7 +86,7 @@ Join condition: None Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(13) RegularHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt index 50fc5e0f6630..b5073901c620 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt @@ -1,44 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (46) +- == Final Plan == - VeloxColumnarToRowExec (30) - +- ^ SortExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- VeloxAppendBatches (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) + VeloxColumnarToRowExec (31) + +- ^ SortExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (21) :- ^ InputIteratorTransformer (7) : +- BroadcastQueryStage (5) : +- ColumnarBroadcastExchange (4) : +- ^ NoopFilter (2) : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- Project (42) - +- BroadcastHashJoin Inner BuildLeft (41) - :- BroadcastExchange (33) - : +- Filter (32) - : +- Scan parquet (31) - +- Filter (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- Filter (35) - +- Scan parquet (34) + Sort (45) + +- Exchange (44) + +- Project (43) + +- BroadcastHashJoin Inner BuildLeft (42) + :- BroadcastExchange (34) + : +- Filter (33) + : +- Scan parquet (32) + +- Filter (41) + +- HashAggregate (40) + +- Exchange (39) + +- HashAggregate (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) (1) Scan parquet @@ -100,142 +101,146 @@ Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) VeloxAppendBatches +(24) VeloxAppendBatches Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(24) ColumnarExchange +(25) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) SortExecTransformer +(29) SortExecTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(29) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(30) VeloxColumnarToRowExec +(31) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(31) Scan parquet +(32) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(32) Filter +(33) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(33) BroadcastExchange +(34) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(34) Scan parquet +(35) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(35) Filter +(36) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(36) Project +(37) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(37) HashAggregate +(38) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(38) Exchange +(39) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(40) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(40) Filter +(41) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(42) Project +(43) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(43) Exchange +(44) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(45) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(46) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt index ed3433681532..f3cef927551c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt @@ -1,54 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- VeloxAppendBatches (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ NoopFilter (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ NoopFilter (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (56) - +- Exchange (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- BroadcastHashJoin LeftAnti BuildRight (43) - : :- Filter (38) - : : +- Scan parquet (37) - : +- BroadcastExchange (42) - : +- Project (41) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Filter (45) - +- Scan parquet (44) + Sort (58) + +- Exchange (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Project (50) + +- BroadcastHashJoin Inner BuildRight (49) + :- BroadcastHashJoin LeftAnti BuildRight (45) + : :- Filter (40) + : : +- Scan parquet (39) + : +- BroadcastExchange (44) + : +- Project (43) + : +- Filter (42) + : +- Scan parquet (41) + +- BroadcastExchange (48) + +- Filter (47) + +- Scan parquet (46) (1) Scan parquet @@ -115,199 +117,207 @@ Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(16) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(25) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(26) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) VeloxAppendBatches +(31) VeloxAppendBatches Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(30) ColumnarExchange +(32) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(32) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(34) SortExecTransformer +(36) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(37) Scan parquet +(39) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(38) Filter +(40) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(39) Scan parquet +(41) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(40) Filter +(42) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(41) Project +(43) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(42) BroadcastExchange +(44) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(43) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(44) Scan parquet +(46) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(45) Filter +(47) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(46) BroadcastExchange +(48) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(47) BroadcastHashJoin +(49) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(48) Project +(50) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(49) HashAggregate +(51) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(50) Exchange +(52) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(52) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(53) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(56) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(55) Exchange +(57) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) Sort +(58) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(57) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/17.txt index 1b258da06f20..7c5359849d4e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/17.txt @@ -8,7 +8,7 @@ AdaptiveSparkPlan (36) +- ShuffleQueryStage (9) +- ColumnarExchange (8) +- VeloxAppendBatches (7) - +- ^ RegularHashAggregateExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (5) +- ^ InputIteratorTransformer (4) +- RowToVeloxColumnar (2) +- LocalTableScan (1) @@ -48,7 +48,7 @@ Input [1]: [l_extendedprice#X] (4) InputIteratorTransformer Input [1]: [l_extendedprice#X] -(5) RegularHashAggregateExecTransformer +(5) FlushableHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt index 505beebed868..a0a6e26a6ac3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt @@ -9,7 +9,7 @@ AdaptiveSparkPlan (88) +- ColumnarExchange (48) +- VeloxAppendBatches (47) +- ^ ProjectExecTransformer (45) - +- ^ RegularHashAggregateExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (44) +- ^ ProjectExecTransformer (43) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) :- ^ ProjectExecTransformer (29) @@ -33,7 +33,7 @@ AdaptiveSparkPlan (88) : +- ColumnarExchange (15) : +- VeloxAppendBatches (14) : +- ^ ProjectExecTransformer (12) - : +- ^ RegularHashAggregateExecTransformer (11) + : +- ^ FlushableHashAggregateExecTransformer (11) : +- ^ Scan parquet (10) +- ^ InputIteratorTransformer (41) +- BroadcastQueryStage (39) @@ -125,7 +125,7 @@ Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) RegularHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] @@ -261,7 +261,7 @@ Join condition: None Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(44) RegularHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt index f4c3b231f1ba..d2ab9979f8a3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt @@ -7,7 +7,7 @@ AdaptiveSparkPlan (34) +- ShuffleQueryStage (17) +- ColumnarExchange (16) +- VeloxAppendBatches (15) - +- ^ RegularHashAggregateExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) @@ -85,7 +85,7 @@ Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) RegularHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt index 426a01b76051..6ebe36be3494 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt @@ -1,16 +1,16 @@ == Physical Plan == -AdaptiveSparkPlan (109) +AdaptiveSparkPlan (110) +- == Final Plan == - VeloxColumnarToRowExec (72) - +- ^ SortExecTransformer (70) - +- ^ InputIteratorTransformer (69) - +- ShuffleQueryStage (67) - +- ColumnarExchange (66) - +- VeloxAppendBatches (65) - +- ^ ProjectExecTransformer (63) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (62) - :- ^ ProjectExecTransformer (53) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (52) + VeloxColumnarToRowExec (73) + +- ^ SortExecTransformer (71) + +- ^ InputIteratorTransformer (70) + +- ShuffleQueryStage (68) + +- ColumnarExchange (67) + +- VeloxAppendBatches (66) + +- ^ ProjectExecTransformer (64) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (63) + :- ^ ProjectExecTransformer (54) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (53) : :- ^ InputIteratorTransformer (10) : : +- AQEShuffleRead (8) : : +- ShuffleQueryStage (7) @@ -19,11 +19,11 @@ AdaptiveSparkPlan (109) : : +- ^ ProjectExecTransformer (3) : : +- ^ NoopFilter (2) : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (51) - : +- BroadcastQueryStage (49) - : +- ColumnarBroadcastExchange (48) - : +- ^ ProjectExecTransformer (46) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (45) + : +- ^ InputIteratorTransformer (52) + : +- BroadcastQueryStage (50) + : +- ColumnarBroadcastExchange (49) + : +- ^ ProjectExecTransformer (47) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (46) : :- ^ InputIteratorTransformer (26) : : +- BroadcastQueryStage (24) : : +- ColumnarBroadcastExchange (23) @@ -36,64 +36,65 @@ AdaptiveSparkPlan (109) : : +- ^ ProjectExecTransformer (15) : : +- ^ NoopFilter (14) : : +- ^ Scan parquet (13) - : +- ^ FilterExecTransformer (44) - : +- ^ ProjectExecTransformer (43) - : +- ^ RegularHashAggregateExecTransformer (42) - : +- ^ InputIteratorTransformer (41) - : +- ShuffleQueryStage (39) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ FlushableHashAggregateExecTransformer (35) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (34) - : :- ^ ProjectExecTransformer (29) - : : +- ^ NoopFilter (28) - : : +- ^ Scan parquet (27) - : +- ^ InputIteratorTransformer (33) - : +- BroadcastQueryStage (31) - : +- ReusedExchange (30) - +- ^ InputIteratorTransformer (61) - +- BroadcastQueryStage (59) - +- ColumnarBroadcastExchange (58) - +- ^ ProjectExecTransformer (56) - +- ^ NoopFilter (55) - +- ^ Scan parquet (54) + : +- ^ FilterExecTransformer (45) + : +- ^ ProjectExecTransformer (44) + : +- ^ RegularHashAggregateExecTransformer (43) + : +- ^ InputIteratorTransformer (42) + : +- ShuffleQueryStage (40) + : +- ColumnarExchange (39) + : +- VeloxAppendBatches (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ FlushableHashAggregateExecTransformer (35) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (34) + : :- ^ ProjectExecTransformer (29) + : : +- ^ NoopFilter (28) + : : +- ^ Scan parquet (27) + : +- ^ InputIteratorTransformer (33) + : +- BroadcastQueryStage (31) + : +- ReusedExchange (30) + +- ^ InputIteratorTransformer (62) + +- BroadcastQueryStage (60) + +- ColumnarBroadcastExchange (59) + +- ^ ProjectExecTransformer (57) + +- ^ NoopFilter (56) + +- ^ Scan parquet (55) +- == Initial Plan == - Sort (108) - +- Exchange (107) - +- Project (106) - +- BroadcastHashJoin Inner BuildRight (105) - :- Project (100) - : +- ShuffledHashJoin LeftSemi BuildRight (99) - : :- Exchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- Exchange (98) - : +- Project (97) - : +- BroadcastHashJoin Inner BuildLeft (96) - : :- BroadcastExchange (83) - : : +- BroadcastHashJoin LeftSemi BuildRight (82) - : : :- Filter (77) - : : : +- Scan parquet (76) - : : +- BroadcastExchange (81) - : : +- Project (80) - : : +- Filter (79) - : : +- Scan parquet (78) - : +- Filter (95) - : +- HashAggregate (94) - : +- Exchange (93) - : +- HashAggregate (92) - : +- BroadcastHashJoin LeftSemi BuildRight (91) - : :- Project (86) - : : +- Filter (85) - : : +- Scan parquet (84) - : +- BroadcastExchange (90) - : +- Project (89) - : +- Filter (88) - : +- Scan parquet (87) - +- BroadcastExchange (104) - +- Project (103) - +- Filter (102) - +- Scan parquet (101) + Sort (109) + +- Exchange (108) + +- Project (107) + +- BroadcastHashJoin Inner BuildRight (106) + :- Project (101) + : +- ShuffledHashJoin LeftSemi BuildRight (100) + : :- Exchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Exchange (99) + : +- Project (98) + : +- BroadcastHashJoin Inner BuildLeft (97) + : :- BroadcastExchange (84) + : : +- BroadcastHashJoin LeftSemi BuildRight (83) + : : :- Filter (78) + : : : +- Scan parquet (77) + : : +- BroadcastExchange (82) + : : +- Project (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- Filter (96) + : +- HashAggregate (95) + : +- Exchange (94) + : +- HashAggregate (93) + : +- BroadcastHashJoin LeftSemi BuildRight (92) + : :- Project (87) + : : +- Filter (86) + : : +- Scan parquet (85) + : +- BroadcastExchange (91) + : +- Project (90) + : +- Filter (89) + : +- Scan parquet (88) + +- BroadcastExchange (105) + +- Project (104) + +- Filter (103) + +- Scan parquet (102) (1) Scan parquet @@ -252,319 +253,323 @@ Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(38) ColumnarExchange +(38) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(39) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(40) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(40) InputAdapter +(41) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(41) InputIteratorTransformer +(42) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(42) RegularHashAggregateExecTransformer +(43) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) ProjectExecTransformer +(44) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(44) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(45) BroadcastHashJoinExecTransformer +(46) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(46) ProjectExecTransformer +(47) ProjectExecTransformer Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(47) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] Arguments: false -(48) ColumnarBroadcastExchange +(49) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(49) BroadcastQueryStage +(50) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [1]: [ps_suppkey#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(52) BroadcastHashJoinExecTransformer +(53) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(53) ProjectExecTransformer +(54) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(54) Scan parquet +(55) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(55) NoopFilter +(56) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(56) ProjectExecTransformer +(57) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(58) ColumnarBroadcastExchange +(59) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(59) BroadcastQueryStage +(60) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(60) InputAdapter +(61) InputAdapter Input [1]: [n_nationkey#X] -(61) InputIteratorTransformer +(62) InputIteratorTransformer Input [1]: [n_nationkey#X] -(62) BroadcastHashJoinExecTransformer +(63) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(64) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(65) VeloxAppendBatches +(66) VeloxAppendBatches Input [2]: [s_name#X, s_address#X] Arguments: X -(66) ColumnarExchange +(67) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(68) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(68) InputAdapter +(69) InputAdapter Input [2]: [s_name#X, s_address#X] -(69) InputIteratorTransformer +(70) InputIteratorTransformer Input [2]: [s_name#X, s_address#X] -(70) SortExecTransformer +(71) SortExecTransformer Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(71) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(72) VeloxColumnarToRowExec +(73) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(73) Scan parquet +(74) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(75) Exchange +(76) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) Scan parquet +(77) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(77) Filter +(78) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(78) Scan parquet +(79) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(79) Filter +(80) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(80) Project +(81) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(81) BroadcastExchange +(82) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(83) BroadcastExchange +(84) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(84) Scan parquet +(85) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(86) Project +(87) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(87) Scan parquet +(88) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(88) Filter +(89) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(89) Project +(90) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(90) BroadcastExchange +(91) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(92) HashAggregate +(93) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(93) Exchange +(94) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) HashAggregate +(95) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(95) Filter +(96) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(96) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(97) Project +(98) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(98) Exchange +(99) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) ShuffledHashJoin +(100) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(100) Project +(101) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(101) Scan parquet +(102) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(102) Filter +(103) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(103) Project +(104) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(104) BroadcastExchange +(105) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(105) BroadcastHashJoin +(106) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(106) Project +(107) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(107) Exchange +(108) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Sort +(109) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(109) AdaptiveSparkPlan +(110) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt index 10cf069932d7..eb3f64d3efcc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt @@ -9,7 +9,7 @@ AdaptiveSparkPlan (93) +- ColumnarExchange (53) +- VeloxAppendBatches (52) +- ^ ProjectExecTransformer (50) - +- ^ RegularHashAggregateExecTransformer (49) + +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) :- ^ ProjectExecTransformer (38) @@ -287,7 +287,7 @@ Join condition: None Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(49) RegularHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt index 287d6c30505a..4c149d9bb43e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt @@ -1,38 +1,39 @@ == Physical Plan == -AdaptiveSparkPlan (39) +AdaptiveSparkPlan (40) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- ^ SortExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22) - +- ColumnarExchange (21) - +- VeloxAppendBatches (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (28) + +- ^ SortExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- VeloxAppendBatches (21) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (38) - +- Exchange (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- BroadcastHashJoin LeftAnti BuildRight (32) - :- Filter (29) - : +- Scan parquet (28) - +- BroadcastExchange (31) - +- Scan parquet (30) + Sort (39) + +- Exchange (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- BroadcastHashJoin LeftAnti BuildRight (33) + :- Filter (30) + : +- Scan parquet (29) + +- BroadcastExchange (32) + +- Scan parquet (31) (1) Scan parquet @@ -94,116 +95,120 @@ Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) VeloxAppendBatches +(21) VeloxAppendBatches Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(21) ColumnarExchange +(22) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(23) InputAdapter +(24) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(25) SortExecTransformer +(26) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(26) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(28) Scan parquet +(29) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(29) Filter +(30) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(30) Scan parquet +(31) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(31) BroadcastExchange +(32) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(32) BroadcastHashJoin +(33) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(33) Project +(34) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(34) HashAggregate +(35) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(35) Exchange +(36) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(37) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(37) Exchange +(38) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Sort +(39) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(39) AdaptiveSparkPlan +(40) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt index d92e785299d4..df2a6de8d184 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt @@ -10,7 +10,7 @@ AdaptiveSparkPlan (54) +- ColumnarExchange (27) +- VeloxAppendBatches (26) +- ^ ProjectExecTransformer (24) - +- ^ RegularHashAggregateExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (23) +- ^ ProjectExecTransformer (22) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) :- ^ ProjectExecTransformer (12) @@ -145,7 +145,7 @@ Join condition: None Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) RegularHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt index 68bfa339986a..34cb6467cbc2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt @@ -1,44 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (46) +- == Final Plan == - VeloxColumnarToRowExec (30) - +- ^ SortExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- VeloxAppendBatches (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (31) + +- ^ SortExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ InputIteratorTransformer (21) + +- ShuffleQueryStage (19) + +- ColumnarExchange (18) + +- VeloxAppendBatches (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- HashAggregate (42) - +- Exchange (41) - +- HashAggregate (40) - +- Project (39) - +- BroadcastHashJoin LeftSemi BuildRight (38) - :- Project (33) - : +- Filter (32) - : +- Scan parquet (31) - +- BroadcastExchange (37) - +- Project (36) - +- Filter (35) - +- Scan parquet (34) + Sort (45) + +- Exchange (44) + +- HashAggregate (43) + +- Exchange (42) + +- HashAggregate (41) + +- Project (40) + +- BroadcastHashJoin LeftSemi BuildRight (39) + :- Project (34) + : +- Filter (33) + : +- Scan parquet (32) + +- BroadcastExchange (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) (1) Scan parquet @@ -113,129 +114,133 @@ Input [2]: [o_orderpriority#X, count#X] Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(17) ColumnarExchange +(17) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(18) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(18) ShuffleQueryStage +(19) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(19) InputAdapter +(20) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(20) InputIteratorTransformer +(21) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(21) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) VeloxAppendBatches +(24) VeloxAppendBatches Input [2]: [o_orderpriority#X, order_count#X] Arguments: X -(24) ColumnarExchange +(25) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(28) SortExecTransformer +(29) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(29) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(30) VeloxColumnarToRowExec +(31) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(31) Scan parquet +(32) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(32) Filter +(33) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(33) Project +(34) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(34) Scan parquet +(35) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(35) Filter +(36) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(36) Project +(37) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(37) BroadcastExchange +(38) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(38) BroadcastHashJoin +(39) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(39) Project +(40) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(40) HashAggregate +(41) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(41) Exchange +(42) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) HashAggregate +(43) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(43) Exchange +(44) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(45) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(46) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt index 612d61aac3b4..2c9ba4c938d4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt @@ -1,92 +1,93 @@ == Physical Plan == -AdaptiveSparkPlan (101) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (66) - +- ^ SortExecTransformer (64) - +- ^ InputIteratorTransformer (63) - +- ShuffleQueryStage (61) - +- ColumnarExchange (60) - +- VeloxAppendBatches (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (67) + +- ^ SortExecTransformer (65) + +- ^ InputIteratorTransformer (64) + +- ShuffleQueryStage (62) + +- ColumnarExchange (61) + +- VeloxAppendBatches (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ InputIteratorTransformer (57) + +- ShuffleQueryStage (55) + +- ColumnarExchange (54) + +- VeloxAppendBatches (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (100) - +- Exchange (99) - +- HashAggregate (98) - +- Exchange (97) - +- HashAggregate (96) - +- Project (95) - +- BroadcastHashJoin Inner BuildRight (94) - :- Project (89) - : +- BroadcastHashJoin Inner BuildRight (88) - : :- Project (84) - : : +- BroadcastHashJoin Inner BuildRight (83) - : : :- Project (79) - : : : +- BroadcastHashJoin Inner BuildRight (78) - : : : :- Project (74) - : : : : +- BroadcastHashJoin Inner BuildLeft (73) - : : : : :- BroadcastExchange (69) - : : : : : +- Filter (68) - : : : : : +- Scan parquet (67) - : : : : +- Project (72) - : : : : +- Filter (71) - : : : : +- Scan parquet (70) - : : : +- BroadcastExchange (77) - : : : +- Filter (76) - : : : +- Scan parquet (75) - : : +- BroadcastExchange (82) - : : +- Filter (81) - : : +- Scan parquet (80) - : +- BroadcastExchange (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (93) - +- Project (92) - +- Filter (91) - +- Scan parquet (90) + Sort (101) + +- Exchange (100) + +- HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Project (96) + +- BroadcastHashJoin Inner BuildRight (95) + :- Project (90) + : +- BroadcastHashJoin Inner BuildRight (89) + : :- Project (85) + : : +- BroadcastHashJoin Inner BuildRight (84) + : : :- Project (80) + : : : +- BroadcastHashJoin Inner BuildRight (79) + : : : :- Project (75) + : : : : +- BroadcastHashJoin Inner BuildLeft (74) + : : : : :- BroadcastExchange (70) + : : : : : +- Filter (69) + : : : : : +- Scan parquet (68) + : : : : +- Project (73) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (78) + : : : +- Filter (77) + : : : +- Scan parquet (76) + : : +- BroadcastExchange (83) + : : +- Filter (82) + : : +- Scan parquet (81) + : +- BroadcastExchange (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (94) + +- Project (93) + +- Filter (92) + +- Scan parquet (91) (1) Scan parquet @@ -313,225 +314,229 @@ Input [3]: [n_name#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(53) ColumnarExchange +(53) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(54) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(55) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(55) InputAdapter +(56) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(56) InputIteratorTransformer +(57) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(57) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(58) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(59) VeloxAppendBatches +(60) VeloxAppendBatches Input [2]: [n_name#X, revenue#X] Arguments: X -(60) ColumnarExchange +(61) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(62) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(62) InputAdapter +(63) InputAdapter Input [2]: [n_name#X, revenue#X] -(63) InputIteratorTransformer +(64) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(64) SortExecTransformer +(65) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(65) WholeStageCodegenTransformer (X) +(66) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(66) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(67) Scan parquet +(68) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(68) Filter +(69) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(69) BroadcastExchange +(70) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(70) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(72) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(74) Project +(75) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(75) Scan parquet +(76) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(76) Filter +(77) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(77) BroadcastExchange +(78) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(78) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(79) Project +(80) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(80) Scan parquet +(81) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(81) Filter +(82) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(82) BroadcastExchange +(83) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(83) BroadcastHashJoin +(84) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(84) Project +(85) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(85) Scan parquet +(86) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(87) BroadcastExchange +(88) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(89) Project +(90) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(90) Scan parquet +(91) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(91) Filter +(92) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(92) Project +(93) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(93) BroadcastExchange +(94) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(94) BroadcastHashJoin +(95) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(95) Project +(96) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(96) HashAggregate +(97) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(97) Exchange +(98) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) HashAggregate +(99) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(99) Exchange +(100) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Sort +(101) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(101) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt index a5022343944f..b6f876d48e5a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt @@ -7,7 +7,7 @@ AdaptiveSparkPlan (20) +- ShuffleQueryStage (8) +- ColumnarExchange (7) +- VeloxAppendBatches (6) - +- ^ RegularHashAggregateExecTransformer (4) + +- ^ FlushableHashAggregateExecTransformer (4) +- ^ ProjectExecTransformer (3) +- ^ NoopFilter (2) +- ^ Scan parquet (1) @@ -35,7 +35,7 @@ Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) RegularHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt index 3ed89ddff162..8cf20f7694c9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt @@ -1,86 +1,87 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (95) +- == Final Plan == - VeloxColumnarToRowExec (61) - +- ^ SortExecTransformer (59) - +- ^ InputIteratorTransformer (58) - +- ShuffleQueryStage (56) - +- ColumnarExchange (55) - +- VeloxAppendBatches (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ NoopFilter (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ NoopFilter (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34) - : +- ColumnarBroadcastExchange (33) - : +- ^ NoopFilter (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40) - +- ReusedExchange (39) + VeloxColumnarToRowExec (62) + +- ^ SortExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57) + +- ColumnarExchange (56) + +- VeloxAppendBatches (55) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ NoopFilter (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ NoopFilter (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40) + +- ReusedExchange (39) +- == Initial Plan == - Sort (93) - +- Exchange (92) - +- HashAggregate (91) - +- Exchange (90) - +- HashAggregate (89) - +- Project (88) - +- BroadcastHashJoin Inner BuildRight (87) - :- Project (83) - : +- BroadcastHashJoin Inner BuildRight (82) - : :- Project (78) - : : +- BroadcastHashJoin Inner BuildRight (77) - : : :- Project (73) - : : : +- BroadcastHashJoin Inner BuildRight (72) - : : : :- Project (68) - : : : : +- BroadcastHashJoin Inner BuildLeft (67) - : : : : :- BroadcastExchange (64) - : : : : : +- Filter (63) - : : : : : +- Scan parquet (62) - : : : : +- Filter (66) - : : : : +- Scan parquet (65) - : : : +- BroadcastExchange (71) - : : : +- Filter (70) - : : : +- Scan parquet (69) - : : +- BroadcastExchange (76) - : : +- Filter (75) - : : +- Scan parquet (74) - : +- BroadcastExchange (81) - : +- Filter (80) - : +- Scan parquet (79) - +- BroadcastExchange (86) - +- Filter (85) - +- Scan parquet (84) + Sort (94) + +- Exchange (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- BroadcastHashJoin Inner BuildRight (88) + :- Project (84) + : +- BroadcastHashJoin Inner BuildRight (83) + : :- Project (79) + : : +- BroadcastHashJoin Inner BuildRight (78) + : : :- Project (74) + : : : +- BroadcastHashJoin Inner BuildRight (73) + : : : :- Project (69) + : : : : +- BroadcastHashJoin Inner BuildLeft (68) + : : : : :- BroadcastExchange (65) + : : : : : +- Filter (64) + : : : : : +- Scan parquet (63) + : : : : +- Filter (67) + : : : : +- Scan parquet (66) + : : : +- BroadcastExchange (72) + : : : +- Filter (71) + : : : +- Scan parquet (70) + : : +- BroadcastExchange (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- BroadcastExchange (82) + : +- Filter (81) + : +- Scan parquet (80) + +- BroadcastExchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -283,217 +284,221 @@ Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(48) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(49) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(50) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(53) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) VeloxAppendBatches +(55) VeloxAppendBatches Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(55) ColumnarExchange +(56) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(56) ShuffleQueryStage +(57) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(57) InputAdapter +(58) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) InputIteratorTransformer +(59) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(59) SortExecTransformer +(60) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(60) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(61) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(62) Scan parquet +(63) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(63) Filter +(64) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(64) BroadcastExchange +(65) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(65) Scan parquet +(66) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(67) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(68) Project +(69) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(69) Scan parquet +(70) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(71) BroadcastExchange +(72) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(73) Project +(74) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(75) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(78) Project +(79) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(79) Scan parquet +(80) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(80) Filter +(81) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(81) BroadcastExchange +(82) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(83) Project +(84) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(84) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(85) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(86) BroadcastExchange +(87) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(88) Project +(89) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(89) HashAggregate +(90) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(90) Exchange +(91) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(92) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(92) Exchange +(93) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Sort +(94) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(94) AdaptiveSparkPlan +(95) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt index f309fbdce259..045e283bd036 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt @@ -1,117 +1,118 @@ == Physical Plan == -AdaptiveSparkPlan (130) +AdaptiveSparkPlan (131) +- == Final Plan == - VeloxColumnarToRowExec (85) - +- ^ SortExecTransformer (83) - +- ^ InputIteratorTransformer (82) - +- ShuffleQueryStage (80) - +- ColumnarExchange (79) - +- VeloxAppendBatches (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ NoopFilter (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ NoopFilter (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ NoopFilter (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ NoopFilter (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53) - : +- ColumnarBroadcastExchange (52) - : +- ^ NoopFilter (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ NoopFilter (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (86) + +- ^ SortExecTransformer (84) + +- ^ InputIteratorTransformer (83) + +- ShuffleQueryStage (81) + +- ColumnarExchange (80) + +- VeloxAppendBatches (79) + +- ^ ProjectExecTransformer (77) + +- ^ RegularHashAggregateExecTransformer (76) + +- ^ InputIteratorTransformer (75) + +- ShuffleQueryStage (73) + +- ColumnarExchange (72) + +- VeloxAppendBatches (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ NoopFilter (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ NoopFilter (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ NoopFilter (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ NoopFilter (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53) + : +- ColumnarBroadcastExchange (52) + : +- ^ NoopFilter (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ NoopFilter (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (129) - +- Exchange (128) - +- HashAggregate (127) - +- Exchange (126) - +- HashAggregate (125) - +- Project (124) - +- BroadcastHashJoin Inner BuildRight (123) - :- Project (118) - : +- BroadcastHashJoin Inner BuildRight (117) - : :- Project (113) - : : +- BroadcastHashJoin Inner BuildRight (112) - : : :- Project (108) - : : : +- BroadcastHashJoin Inner BuildRight (107) - : : : :- Project (103) - : : : : +- BroadcastHashJoin Inner BuildRight (102) - : : : : :- Project (98) - : : : : : +- BroadcastHashJoin Inner BuildRight (97) - : : : : : :- Project (93) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (92) - : : : : : : :- BroadcastExchange (89) - : : : : : : : +- Project (88) - : : : : : : : +- Filter (87) - : : : : : : : +- Scan parquet (86) - : : : : : : +- Filter (91) - : : : : : : +- Scan parquet (90) - : : : : : +- BroadcastExchange (96) - : : : : : +- Filter (95) - : : : : : +- Scan parquet (94) - : : : : +- BroadcastExchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- BroadcastExchange (106) - : : : +- Filter (105) - : : : +- Scan parquet (104) - : : +- BroadcastExchange (111) - : : +- Filter (110) - : : +- Scan parquet (109) - : +- BroadcastExchange (116) - : +- Filter (115) - : +- Scan parquet (114) - +- BroadcastExchange (122) - +- Project (121) - +- Filter (120) - +- Scan parquet (119) + Sort (130) + +- Exchange (129) + +- HashAggregate (128) + +- Exchange (127) + +- HashAggregate (126) + +- Project (125) + +- BroadcastHashJoin Inner BuildRight (124) + :- Project (119) + : +- BroadcastHashJoin Inner BuildRight (118) + : :- Project (114) + : : +- BroadcastHashJoin Inner BuildRight (113) + : : :- Project (109) + : : : +- BroadcastHashJoin Inner BuildRight (108) + : : : :- Project (104) + : : : : +- BroadcastHashJoin Inner BuildRight (103) + : : : : :- Project (99) + : : : : : +- BroadcastHashJoin Inner BuildRight (98) + : : : : : :- Project (94) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (93) + : : : : : : :- BroadcastExchange (90) + : : : : : : : +- Project (89) + : : : : : : : +- Filter (88) + : : : : : : : +- Scan parquet (87) + : : : : : : +- Filter (92) + : : : : : : +- Scan parquet (91) + : : : : : +- BroadcastExchange (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- BroadcastExchange (102) + : : : : +- Filter (101) + : : : : +- Scan parquet (100) + : : : +- BroadcastExchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- BroadcastExchange (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- BroadcastExchange (117) + : +- Filter (116) + : +- Scan parquet (115) + +- BroadcastExchange (123) + +- Project (122) + +- Filter (121) + +- Scan parquet (120) (1) Scan parquet @@ -414,277 +415,281 @@ Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(71) ColumnarExchange +(71) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(72) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(72) ShuffleQueryStage +(73) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(73) InputAdapter +(74) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(74) InputIteratorTransformer +(75) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(75) RegularHashAggregateExecTransformer +(76) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(76) ProjectExecTransformer +(77) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(77) WholeStageCodegenTransformer (X) +(78) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) VeloxAppendBatches +(79) VeloxAppendBatches Input [2]: [o_year#X, mkt_share#X] Arguments: X -(79) ColumnarExchange +(80) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(80) ShuffleQueryStage +(81) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(81) InputAdapter +(82) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(82) InputIteratorTransformer +(83) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(83) SortExecTransformer +(84) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(84) WholeStageCodegenTransformer (X) +(85) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(85) VeloxColumnarToRowExec +(86) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(86) Scan parquet +(87) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(87) Filter +(88) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(88) Project +(89) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(89) BroadcastExchange +(90) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(90) Scan parquet +(91) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(91) Filter +(92) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(92) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(93) Project +(94) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(94) Scan parquet +(95) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(95) Filter +(96) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(96) BroadcastExchange +(97) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(97) BroadcastHashJoin +(98) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(98) Project +(99) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(99) Scan parquet +(100) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(100) Filter +(101) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(101) BroadcastExchange +(102) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(102) BroadcastHashJoin +(103) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(103) Project +(104) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(104) Scan parquet +(105) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(105) Filter +(106) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(106) BroadcastExchange +(107) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(107) BroadcastHashJoin +(108) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(108) Project +(109) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(109) Scan parquet +(110) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(110) Filter +(111) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(111) BroadcastExchange +(112) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(112) BroadcastHashJoin +(113) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(113) Project +(114) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(114) Scan parquet +(115) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(115) Filter +(116) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(116) BroadcastExchange +(117) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(117) BroadcastHashJoin +(118) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(118) Project +(119) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(119) Scan parquet +(120) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(120) Filter +(121) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(121) Project +(122) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(122) BroadcastExchange +(123) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(123) BroadcastHashJoin +(124) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(124) Project +(125) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(125) HashAggregate +(126) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(126) Exchange +(127) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) HashAggregate +(128) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(128) Exchange +(129) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Sort +(130) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(130) AdaptiveSparkPlan +(131) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt index 0420b98c2c1f..abdd1d7b9369 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt @@ -1,90 +1,91 @@ == Physical Plan == -AdaptiveSparkPlan (99) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60) - +- ColumnarExchange (59) - +- VeloxAppendBatches (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44) - +- ColumnarBroadcastExchange (43) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (98) - +- Exchange (97) - +- HashAggregate (96) - +- Exchange (95) - +- HashAggregate (94) - +- Project (93) - +- BroadcastHashJoin Inner BuildRight (92) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (69) - : : : : : +- Project (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Filter (71) - : : : : +- Scan parquet (70) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (91) - +- Filter (90) - +- Scan parquet (89) + Sort (99) + +- Exchange (98) + +- HashAggregate (97) + +- Exchange (96) + +- HashAggregate (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (70) + : : : : : +- Project (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -307,221 +308,225 @@ Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(53) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) VeloxAppendBatches +(59) VeloxAppendBatches Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(59) ColumnarExchange +(60) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(61) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(61) InputAdapter +(62) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) InputIteratorTransformer +(63) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(63) SortExecTransformer +(64) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(65) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(67) Filter +(68) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(68) Project +(69) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(69) BroadcastExchange +(70) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(70) Scan parquet +(71) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(73) Project +(74) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(74) Scan parquet +(75) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(78) Project +(79) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(79) Scan parquet +(80) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(80) Filter +(81) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(81) BroadcastExchange +(82) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(83) Project +(84) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(84) Scan parquet +(85) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(86) BroadcastExchange +(87) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(88) Project +(89) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(89) Scan parquet +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(91) BroadcastExchange +(92) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(92) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(93) Project +(94) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(94) HashAggregate +(95) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(95) Exchange +(96) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) HashAggregate +(97) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(97) Exchange +(98) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Sort +(99) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(99) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt index 5ad7bc9d3675..afea15af53d2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (29) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (28) - +- Exchange (27) - +- HashAggregate (26) - +- Exchange (25) - +- HashAggregate (24) - +- Project (23) - +- Filter (22) - +- Scan parquet (21) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -57,101 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) VeloxAppendBatches +(14) VeloxAppendBatches Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) ColumnarExchange +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(18) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(20) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(21) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(22) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(23) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(24) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(25) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(27) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(29) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt index 4e657eb7e540..7a509e435d07 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt @@ -10,7 +10,7 @@ AdaptiveSparkPlan (68) +- ColumnarExchange (36) +- VeloxAppendBatches (35) +- ^ ProjectExecTransformer (33) - +- ^ RegularHashAggregateExecTransformer (32) + +- ^ FlushableHashAggregateExecTransformer (32) +- ^ ProjectExecTransformer (31) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) :- ^ ProjectExecTransformer (22) @@ -195,7 +195,7 @@ Join condition: None Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(32) RegularHashAggregateExecTransformer +(32) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt index 5be334bc198e..1c4e102aa0b9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt @@ -1,56 +1,57 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- ^ SortExecTransformer (37) - +- ^ InputIteratorTransformer (36) - +- ShuffleQueryStage (34), Statistics(X) - +- ColumnarExchange (33) - +- VeloxAppendBatches (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7), Statistics(X) - : +- ColumnarBroadcastExchange (6) - : +- ^ NoopFilter (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17), Statistics(X) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ NoopFilter (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ FilterExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ NoopFilter (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ NoopFilter (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (58) - +- Exchange (57) - +- Filter (56) - +- HashAggregate (55) - +- Exchange (54) - +- HashAggregate (53) - +- Project (52) - +- BroadcastHashJoin Inner BuildRight (51) - :- Project (46) - : +- BroadcastHashJoin Inner BuildRight (45) - : :- Filter (41) - : : +- Scan parquet (40) - : +- BroadcastExchange (44) - : +- Filter (43) - : +- Scan parquet (42) - +- BroadcastExchange (50) - +- Project (49) - +- Filter (48) - +- Scan parquet (47) + Sort (59) + +- Exchange (58) + +- Filter (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- BroadcastHashJoin Inner BuildRight (52) + :- Project (47) + : +- BroadcastHashJoin Inner BuildRight (46) + : :- Filter (42) + : : +- Scan parquet (41) + : +- BroadcastExchange (45) + : +- Filter (44) + : +- Scan parquet (43) + +- BroadcastExchange (51) + +- Project (50) + +- Filter (49) + +- Scan parquet (48) (1) Scan parquet @@ -159,383 +160,392 @@ Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(25) ColumnarExchange +(25) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(26) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(27) InputAdapter +(28) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(28) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(29) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(30) FilterExecTransformer +(31) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) VeloxAppendBatches +(33) VeloxAppendBatches Input [2]: [ps_partkey#X, value#X] Arguments: X -(33) ColumnarExchange +(34) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(34) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(35) InputAdapter +(36) InputAdapter Input [2]: [ps_partkey#X, value#X] -(36) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(37) SortExecTransformer +(38) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(38) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(39) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(40) Scan parquet +(41) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(41) Filter +(42) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(42) Scan parquet +(43) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(43) Filter +(44) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(44) BroadcastExchange +(45) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(45) BroadcastHashJoin +(46) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(46) Project +(47) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(47) Scan parquet +(48) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(48) Filter +(49) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(49) Project +(50) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(50) BroadcastExchange +(51) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(51) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(52) Project +(53) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(53) HashAggregate +(54) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(54) Exchange +(55) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) HashAggregate +(56) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(56) Filter +(57) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(57) Exchange +(58) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) Sort +(59) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(59) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 30 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (100) +Subquery:1 Hosting operator id = 31 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (83) - +- ^ ProjectExecTransformer (81) - +- ^ RegularHashAggregateExecTransformer (80) - +- ^ InputIteratorTransformer (79) - +- ShuffleQueryStage (77), Statistics(X) - +- ColumnarExchange (76) - +- ^ FlushableHashAggregateExecTransformer (74) - +- ^ ProjectExecTransformer (73) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (72) - :- ^ ProjectExecTransformer (67) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - : :- ^ NoopFilter (61) - : : +- ^ Scan parquet (60) - : +- ^ InputIteratorTransformer (65) - : +- BroadcastQueryStage (63), Statistics(X) - : +- ReusedExchange (62) - +- ^ InputIteratorTransformer (71) - +- BroadcastQueryStage (69), Statistics(X) - +- ReusedExchange (68) + VeloxColumnarToRowExec (85) + +- ^ ProjectExecTransformer (83) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- VeloxAppendBatches (77) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (73) + :- ^ ProjectExecTransformer (68) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (67) + : :- ^ NoopFilter (62) + : : +- ^ Scan parquet (61) + : +- ^ InputIteratorTransformer (66) + : +- BroadcastQueryStage (64), Statistics(X) + : +- ReusedExchange (63) + +- ^ InputIteratorTransformer (72) + +- BroadcastQueryStage (70), Statistics(X) + +- ReusedExchange (69) +- == Initial Plan == - HashAggregate (99) - +- Exchange (98) - +- HashAggregate (97) - +- Project (96) - +- BroadcastHashJoin Inner BuildRight (95) - :- Project (90) - : +- BroadcastHashJoin Inner BuildRight (89) - : :- Filter (85) - : : +- Scan parquet (84) - : +- BroadcastExchange (88) - : +- Filter (87) - : +- Scan parquet (86) - +- BroadcastExchange (94) - +- Project (93) - +- Filter (92) - +- Scan parquet (91) - - -(60) Scan parquet + HashAggregate (101) + +- Exchange (100) + +- HashAggregate (99) + +- Project (98) + +- BroadcastHashJoin Inner BuildRight (97) + :- Project (92) + : +- BroadcastHashJoin Inner BuildRight (91) + : :- Filter (87) + : : +- Scan parquet (86) + : +- BroadcastExchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- BroadcastExchange (96) + +- Project (95) + +- Filter (94) + +- Scan parquet (93) + + +(61) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(61) NoopFilter +(62) NoopFilter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(62) ReusedExchange [Reuses operator id: 6] +(63) ReusedExchange [Reuses operator id: 6] Output [2]: [s_suppkey#X, s_nationkey#X] -(63) BroadcastQueryStage +(64) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(64) InputAdapter +(65) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(65) InputIteratorTransformer +(66) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(66) BroadcastHashJoinExecTransformer +(67) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(67) ProjectExecTransformer +(68) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(68) ReusedExchange [Reuses operator id: 16] +(69) ReusedExchange [Reuses operator id: 16] Output [1]: [n_nationkey#X] -(69) BroadcastQueryStage +(70) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(70) InputAdapter +(71) InputAdapter Input [1]: [n_nationkey#X] -(71) InputIteratorTransformer +(72) InputIteratorTransformer Input [1]: [n_nationkey#X] -(72) BroadcastHashJoinExecTransformer +(73) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(73) ProjectExecTransformer +(74) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(74) FlushableHashAggregateExecTransformer +(75) FlushableHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(75) WholeStageCodegenTransformer (X) +(76) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(76) ColumnarExchange +(77) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(78) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(77) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(78) InputAdapter +(80) InputAdapter Input [2]: [sum#X, isEmpty#X] -(79) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(80) RegularHashAggregateExecTransformer +(82) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(81) ProjectExecTransformer +(83) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(82) WholeStageCodegenTransformer (X) +(84) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(83) VeloxColumnarToRowExec +(85) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(84) Scan parquet +(86) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(85) Filter +(87) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(86) Scan parquet +(88) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(87) Filter +(89) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(88) BroadcastExchange +(90) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(89) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(90) Project +(92) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(91) Scan parquet +(93) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(92) Filter +(94) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(93) Project +(95) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(94) BroadcastExchange +(96) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(95) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(96) Project +(98) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(97) HashAggregate +(99) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(98) Exchange +(100) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(99) HashAggregate +(101) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(100) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt index 7104a3e8bc38..12d0f6f0ff95 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt @@ -1,42 +1,43 @@ == Physical Plan == -AdaptiveSparkPlan (43) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- VeloxAppendBatches (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (42) - +- Exchange (41) - +- HashAggregate (40) - +- Exchange (39) - +- HashAggregate (38) - +- Project (37) - +- BroadcastHashJoin Inner BuildLeft (36) - :- BroadcastExchange (32) - : +- Filter (31) - : +- Scan parquet (30) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (43) + +- Exchange (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- BroadcastHashJoin Inner BuildLeft (37) + :- BroadcastExchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -107,125 +108,129 @@ Input [3]: [l_shipmode#X, sum#X, sum#X] Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(16) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(20) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) VeloxAppendBatches +(23) VeloxAppendBatches Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(23) ColumnarExchange +(24) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(27) SortExecTransformer +(28) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(30) Scan parquet +(31) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(32) BroadcastExchange +(33) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(33) Scan parquet +(34) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(34) Filter +(35) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(35) Project +(36) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(36) BroadcastHashJoin +(37) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(37) Project +(38) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(38) HashAggregate +(39) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(39) Exchange +(40) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) HashAggregate +(41) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(41) Exchange +(42) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Sort +(43) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(43) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt index 9658b4be0cc7..1dfb80d78050 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt @@ -1,50 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32), Statistics(X) - +- ColumnarExchange (31) - +- VeloxAppendBatches (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ NoopFilter (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ NoopFilter (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (52) - +- Exchange (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- BroadcastHashJoin LeftOuter BuildRight (43) - :- Scan parquet (38) - +- BroadcastExchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- BroadcastHashJoin LeftOuter BuildRight (45) + :- Scan parquet (40) + +- BroadcastExchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -110,178 +112,186 @@ Input [2]: [c_custkey#X, count#X] Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, count#X] +Arguments: X + +(16) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [c_custkey#X, count#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [c_custkey#X, count#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(22) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) VeloxAppendBatches +(32) VeloxAppendBatches Input [2]: [c_count#X, custdist#X] Arguments: X -(31) ColumnarExchange +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(33) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(34) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(35) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(36) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(37) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(38) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(39) Scan parquet +(41) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(42) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(43) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) BroadcastExchange +(44) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(43) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(46) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) Exchange +(48) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(48) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(49) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(51) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(53) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt index fbcaf99758c4..6f71859cba6d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt @@ -8,7 +8,7 @@ AdaptiveSparkPlan (35) +- ShuffleQueryStage (17), Statistics(X) +- ColumnarExchange (16) +- VeloxAppendBatches (15) - +- ^ RegularHashAggregateExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) @@ -86,7 +86,7 @@ Join condition: None Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(13) RegularHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt index 91b117a86c00..40b342510de2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt @@ -1,43 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (43) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- AQEShuffleRead (26) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- VeloxAppendBatches (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) + VeloxColumnarToRowExec (28) + +- AQEShuffleRead (27) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (21) :- ^ InputIteratorTransformer (7) : +- BroadcastQueryStage (5), Statistics(X) : +- ColumnarBroadcastExchange (4) : +- ^ NoopFilter (2) : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- Project (39) - +- BroadcastHashJoin Inner BuildLeft (38) - :- BroadcastExchange (30) - : +- Filter (29) - : +- Scan parquet (28) - +- Filter (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (42) + +- Exchange (41) + +- Project (40) + +- BroadcastHashJoin Inner BuildLeft (39) + :- BroadcastExchange (31) + : +- Filter (30) + : +- Scan parquet (29) + +- Filter (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- Filter (33) + +- Scan parquet (32) (1) Scan parquet @@ -99,288 +100,297 @@ Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) VeloxAppendBatches +(24) VeloxAppendBatches Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(24) ColumnarExchange +(25) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) AQEShuffleRead +(27) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(27) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) Scan parquet +(29) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(29) Filter +(30) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(30) BroadcastExchange +(31) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(31) Scan parquet +(32) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(32) Filter +(33) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(33) Project +(34) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(34) HashAggregate +(35) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(35) Exchange +(36) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(37) Filter +(38) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(38) BroadcastHashJoin +(39) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(39) Project +(40) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(40) Exchange +(41) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(42) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(43) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 19 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (67) +Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ RegularHashAggregateExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ ProjectExecTransformer (47) - +- ^ FlushableHashAggregateExecTransformer (46) - +- ^ ProjectExecTransformer (45) - +- ^ NoopFilter (44) - +- ^ Scan parquet (43) + VeloxColumnarToRowExec (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ ProjectExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ InputIteratorTransformer (54) + +- ShuffleQueryStage (52), Statistics(X) + +- ColumnarExchange (51) + +- VeloxAppendBatches (50) + +- ^ ProjectExecTransformer (48) + +- ^ FlushableHashAggregateExecTransformer (47) + +- ^ ProjectExecTransformer (46) + +- ^ NoopFilter (45) + +- ^ Scan parquet (44) +- == Initial Plan == - HashAggregate (66) - +- HashAggregate (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (68) + +- HashAggregate (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- Filter (62) + +- Scan parquet (61) -(43) Scan parquet +(44) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(44) NoopFilter +(45) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(45) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) FlushableHashAggregateExecTransformer +(47) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(48) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(49) ColumnarExchange +(50) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(51) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(51) InputAdapter +(53) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(53) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(54) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(55) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(56) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(57) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(58) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(59) Scan parquet +(61) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(60) Filter +(62) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(61) Project +(63) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(63) Exchange +(65) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(65) HashAggregate +(67) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(66) HashAggregate +(68) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(67) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt index da533a30dde7..d579c1a704e5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt @@ -1,54 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- VeloxAppendBatches (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ NoopFilter (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ NoopFilter (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (56) - +- Exchange (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- BroadcastHashJoin LeftAnti BuildRight (43) - : :- Filter (38) - : : +- Scan parquet (37) - : +- BroadcastExchange (42) - : +- Project (41) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Filter (45) - +- Scan parquet (44) + Sort (58) + +- Exchange (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Project (50) + +- BroadcastHashJoin Inner BuildRight (49) + :- BroadcastHashJoin LeftAnti BuildRight (45) + : :- Filter (40) + : : +- Scan parquet (39) + : +- BroadcastExchange (44) + : +- Project (43) + : +- Filter (42) + : +- Scan parquet (41) + +- BroadcastExchange (48) + +- Filter (47) + +- Scan parquet (46) (1) Scan parquet @@ -115,199 +117,207 @@ Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(16) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(25) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(26) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) VeloxAppendBatches +(31) VeloxAppendBatches Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(30) ColumnarExchange +(32) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(32) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(34) SortExecTransformer +(36) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(37) Scan parquet +(39) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(38) Filter +(40) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(39) Scan parquet +(41) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(40) Filter +(42) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(41) Project +(43) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(42) BroadcastExchange +(44) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(43) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(44) Scan parquet +(46) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(45) Filter +(47) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(46) BroadcastExchange +(48) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(47) BroadcastHashJoin +(49) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(48) Project +(50) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(49) HashAggregate +(51) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(50) Exchange +(52) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(52) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(53) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(56) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(55) Exchange +(57) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) Sort +(58) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(57) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/17.txt index a57c908f0178..1e2ed970aef4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/17.txt @@ -8,7 +8,7 @@ AdaptiveSparkPlan (36) +- ShuffleQueryStage (9), Statistics(X) +- ColumnarExchange (8) +- VeloxAppendBatches (7) - +- ^ RegularHashAggregateExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (5) +- ^ InputIteratorTransformer (4) +- RowToVeloxColumnar (2) +- LocalTableScan (1) @@ -48,7 +48,7 @@ Input [1]: [l_extendedprice#X] (4) InputIteratorTransformer Input [1]: [l_extendedprice#X] -(5) RegularHashAggregateExecTransformer +(5) FlushableHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt index 810b57c0f9bb..2e7ce455aebd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt @@ -9,7 +9,7 @@ AdaptiveSparkPlan (88) +- ColumnarExchange (48) +- VeloxAppendBatches (47) +- ^ ProjectExecTransformer (45) - +- ^ RegularHashAggregateExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (44) +- ^ ProjectExecTransformer (43) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) :- ^ ProjectExecTransformer (29) @@ -33,7 +33,7 @@ AdaptiveSparkPlan (88) : +- ColumnarExchange (15) : +- VeloxAppendBatches (14) : +- ^ ProjectExecTransformer (12) - : +- ^ RegularHashAggregateExecTransformer (11) + : +- ^ FlushableHashAggregateExecTransformer (11) : +- ^ Scan parquet (10) +- ^ InputIteratorTransformer (41) +- BroadcastQueryStage (39), Statistics(X) @@ -125,7 +125,7 @@ Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) RegularHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] @@ -261,7 +261,7 @@ Join condition: None Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(44) RegularHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt index 46bbc58abcde..992c5328adba 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt @@ -7,7 +7,7 @@ AdaptiveSparkPlan (34) +- ShuffleQueryStage (17), Statistics(X) +- ColumnarExchange (16) +- VeloxAppendBatches (15) - +- ^ RegularHashAggregateExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) @@ -85,7 +85,7 @@ Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) RegularHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt index 3ac12224cffa..27e59afbb7fc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt @@ -1,15 +1,15 @@ == Physical Plan == -AdaptiveSparkPlan (106) +AdaptiveSparkPlan (107) +- == Final Plan == - VeloxColumnarToRowExec (69) - +- AQEShuffleRead (68) - +- ShuffleQueryStage (67), Statistics(X) - +- ColumnarExchange (66) - +- VeloxAppendBatches (65) - +- ^ ProjectExecTransformer (63) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (62) - :- ^ ProjectExecTransformer (53) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (52) + VeloxColumnarToRowExec (70) + +- AQEShuffleRead (69) + +- ShuffleQueryStage (68), Statistics(X) + +- ColumnarExchange (67) + +- VeloxAppendBatches (66) + +- ^ ProjectExecTransformer (64) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (63) + :- ^ ProjectExecTransformer (54) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (53) : :- ^ InputIteratorTransformer (10) : : +- AQEShuffleRead (8) : : +- ShuffleQueryStage (7), Statistics(X) @@ -18,11 +18,11 @@ AdaptiveSparkPlan (106) : : +- ^ ProjectExecTransformer (3) : : +- ^ NoopFilter (2) : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (51) - : +- BroadcastQueryStage (49), Statistics(X) - : +- ColumnarBroadcastExchange (48) - : +- ^ ProjectExecTransformer (46) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (45) + : +- ^ InputIteratorTransformer (52) + : +- BroadcastQueryStage (50), Statistics(X) + : +- ColumnarBroadcastExchange (49) + : +- ^ ProjectExecTransformer (47) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (46) : :- ^ InputIteratorTransformer (26) : : +- BroadcastQueryStage (24), Statistics(X) : : +- ColumnarBroadcastExchange (23) @@ -35,64 +35,65 @@ AdaptiveSparkPlan (106) : : +- ^ ProjectExecTransformer (15) : : +- ^ NoopFilter (14) : : +- ^ Scan parquet (13) - : +- ^ FilterExecTransformer (44) - : +- ^ ProjectExecTransformer (43) - : +- ^ RegularHashAggregateExecTransformer (42) - : +- ^ InputIteratorTransformer (41) - : +- ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ FlushableHashAggregateExecTransformer (35) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (34) - : :- ^ ProjectExecTransformer (29) - : : +- ^ NoopFilter (28) - : : +- ^ Scan parquet (27) - : +- ^ InputIteratorTransformer (33) - : +- BroadcastQueryStage (31), Statistics(X) - : +- ReusedExchange (30) - +- ^ InputIteratorTransformer (61) - +- BroadcastQueryStage (59), Statistics(X) - +- ColumnarBroadcastExchange (58) - +- ^ ProjectExecTransformer (56) - +- ^ NoopFilter (55) - +- ^ Scan parquet (54) + : +- ^ FilterExecTransformer (45) + : +- ^ ProjectExecTransformer (44) + : +- ^ RegularHashAggregateExecTransformer (43) + : +- ^ InputIteratorTransformer (42) + : +- ShuffleQueryStage (40), Statistics(X) + : +- ColumnarExchange (39) + : +- VeloxAppendBatches (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ FlushableHashAggregateExecTransformer (35) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (34) + : :- ^ ProjectExecTransformer (29) + : : +- ^ NoopFilter (28) + : : +- ^ Scan parquet (27) + : +- ^ InputIteratorTransformer (33) + : +- BroadcastQueryStage (31), Statistics(X) + : +- ReusedExchange (30) + +- ^ InputIteratorTransformer (62) + +- BroadcastQueryStage (60), Statistics(X) + +- ColumnarBroadcastExchange (59) + +- ^ ProjectExecTransformer (57) + +- ^ NoopFilter (56) + +- ^ Scan parquet (55) +- == Initial Plan == - Sort (105) - +- Exchange (104) - +- Project (103) - +- BroadcastHashJoin Inner BuildRight (102) - :- Project (97) - : +- ShuffledHashJoin LeftSemi BuildRight (96) - : :- Exchange (72) - : : +- Filter (71) - : : +- Scan parquet (70) - : +- Exchange (95) - : +- Project (94) - : +- BroadcastHashJoin Inner BuildLeft (93) - : :- BroadcastExchange (80) - : : +- BroadcastHashJoin LeftSemi BuildRight (79) - : : :- Filter (74) - : : : +- Scan parquet (73) - : : +- BroadcastExchange (78) - : : +- Project (77) - : : +- Filter (76) - : : +- Scan parquet (75) - : +- Filter (92) - : +- HashAggregate (91) - : +- Exchange (90) - : +- HashAggregate (89) - : +- BroadcastHashJoin LeftSemi BuildRight (88) - : :- Project (83) - : : +- Filter (82) - : : +- Scan parquet (81) - : +- BroadcastExchange (87) - : +- Project (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (101) - +- Project (100) - +- Filter (99) - +- Scan parquet (98) + Sort (106) + +- Exchange (105) + +- Project (104) + +- BroadcastHashJoin Inner BuildRight (103) + :- Project (98) + : +- ShuffledHashJoin LeftSemi BuildRight (97) + : :- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (96) + : +- Project (95) + : +- BroadcastHashJoin Inner BuildLeft (94) + : :- BroadcastExchange (81) + : : +- BroadcastHashJoin LeftSemi BuildRight (80) + : : :- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (79) + : : +- Project (78) + : : +- Filter (77) + : : +- Scan parquet (76) + : +- Filter (93) + : +- HashAggregate (92) + : +- Exchange (91) + : +- HashAggregate (90) + : +- BroadcastHashJoin LeftSemi BuildRight (89) + : :- Project (84) + : : +- Filter (83) + : : +- Scan parquet (82) + : +- BroadcastExchange (88) + : +- Project (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (102) + +- Project (101) + +- Filter (100) + +- Scan parquet (99) (1) Scan parquet @@ -251,309 +252,313 @@ Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(38) ColumnarExchange +(38) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(39) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(40) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(40) InputAdapter +(41) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(41) InputIteratorTransformer +(42) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(42) RegularHashAggregateExecTransformer +(43) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) ProjectExecTransformer +(44) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(44) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(45) BroadcastHashJoinExecTransformer +(46) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(46) ProjectExecTransformer +(47) ProjectExecTransformer Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(47) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] Arguments: false -(48) ColumnarBroadcastExchange +(49) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(49) BroadcastQueryStage +(50) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [1]: [ps_suppkey#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(52) BroadcastHashJoinExecTransformer +(53) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(53) ProjectExecTransformer +(54) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(54) Scan parquet +(55) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(55) NoopFilter +(56) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(56) ProjectExecTransformer +(57) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(58) ColumnarBroadcastExchange +(59) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(59) BroadcastQueryStage +(60) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(60) InputAdapter +(61) InputAdapter Input [1]: [n_nationkey#X] -(61) InputIteratorTransformer +(62) InputIteratorTransformer Input [1]: [n_nationkey#X] -(62) BroadcastHashJoinExecTransformer +(63) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(64) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(65) VeloxAppendBatches +(66) VeloxAppendBatches Input [2]: [s_name#X, s_address#X] Arguments: X -(66) ColumnarExchange +(67) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(68) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(68) AQEShuffleRead +(69) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(69) VeloxColumnarToRowExec +(70) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(70) Scan parquet +(71) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(72) Exchange +(73) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(73) Scan parquet +(74) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(75) Scan parquet +(76) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(76) Filter +(77) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(77) Project +(78) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(78) BroadcastExchange +(79) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(79) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(80) BroadcastExchange +(81) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(81) Scan parquet +(82) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(82) Filter +(83) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(83) Project +(84) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(84) Scan parquet +(85) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(85) Filter +(86) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(86) Project +(87) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(87) BroadcastExchange +(88) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(88) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(89) HashAggregate +(90) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(90) Exchange +(91) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(92) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(92) Filter +(93) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(93) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(94) Project +(95) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(95) Exchange +(96) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(97) Project +(98) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(98) Scan parquet +(99) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(99) Filter +(100) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(100) Project +(101) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(101) BroadcastExchange +(102) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(102) BroadcastHashJoin +(103) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(103) Project +(104) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(104) Exchange +(105) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Sort +(106) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(106) AdaptiveSparkPlan +(107) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt index 9f48891225fb..7627c7f4f147 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt @@ -8,7 +8,7 @@ AdaptiveSparkPlan (92) +- ColumnarExchange (53) +- VeloxAppendBatches (52) +- ^ ProjectExecTransformer (50) - +- ^ RegularHashAggregateExecTransformer (49) + +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) :- ^ ProjectExecTransformer (38) @@ -286,7 +286,7 @@ Join condition: None Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(49) RegularHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt index 404a3debc42c..9a0475d25d78 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt @@ -1,38 +1,39 @@ == Physical Plan == -AdaptiveSparkPlan (39) +AdaptiveSparkPlan (40) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- ^ SortExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- VeloxAppendBatches (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (28) + +- ^ SortExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- VeloxAppendBatches (21) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (38) - +- Exchange (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- BroadcastHashJoin LeftAnti BuildRight (32) - :- Filter (29) - : +- Scan parquet (28) - +- BroadcastExchange (31) - +- Scan parquet (30) + Sort (39) + +- Exchange (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- BroadcastHashJoin LeftAnti BuildRight (33) + :- Filter (30) + : +- Scan parquet (29) + +- BroadcastExchange (32) + +- Scan parquet (31) (1) Scan parquet @@ -94,230 +95,239 @@ Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) VeloxAppendBatches +(21) VeloxAppendBatches Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(21) ColumnarExchange +(22) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(23) InputAdapter +(24) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(25) SortExecTransformer +(26) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(26) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(28) Scan parquet +(29) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(29) Filter +(30) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(30) Scan parquet +(31) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(31) BroadcastExchange +(32) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(32) BroadcastHashJoin +(33) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(33) Project +(34) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(34) HashAggregate +(35) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(35) Exchange +(36) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(37) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(37) Exchange +(38) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Sort +(39) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(39) AdaptiveSparkPlan +(40) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (51) - +- ^ RegularHashAggregateExecTransformer (49) - +- ^ InputIteratorTransformer (48) - +- ShuffleQueryStage (46), Statistics(X) - +- ColumnarExchange (45) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (53) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ShuffleQueryStage (48), Statistics(X) + +- ColumnarExchange (47) + +- VeloxAppendBatches (46) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ NoopFilter (42) + +- ^ Scan parquet (41) +- == Initial Plan == - HashAggregate (57) - +- Exchange (56) - +- HashAggregate (55) - +- Project (54) - +- Filter (53) - +- Scan parquet (52) + HashAggregate (59) + +- Exchange (58) + +- HashAggregate (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) -(40) Scan parquet +(41) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(41) NoopFilter +(42) NoopFilter Input [2]: [c_phone#X, c_acctbal#X] Arguments: [c_phone#X, c_acctbal#X] -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(43) FlushableHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(44) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(45) ColumnarExchange +(46) VeloxAppendBatches +Input [2]: [sum#X, count#X] +Arguments: X + +(47) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(46) ShuffleQueryStage +(48) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(47) InputAdapter +(49) InputAdapter Input [2]: [sum#X, count#X] -(48) InputIteratorTransformer +(50) InputIteratorTransformer Input [2]: [sum#X, count#X] -(49) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(50) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(51) VeloxColumnarToRowExec +(53) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(52) Scan parquet +(54) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(53) Filter +(55) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(54) Project +(56) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(55) HashAggregate +(57) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(56) Exchange +(58) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(57) HashAggregate +(59) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(58) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt index aacc67b90b3f..60843c0991ad 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt @@ -10,7 +10,7 @@ AdaptiveSparkPlan (54) +- ColumnarExchange (27) +- VeloxAppendBatches (26) +- ^ ProjectExecTransformer (24) - +- ^ RegularHashAggregateExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (23) +- ^ ProjectExecTransformer (22) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) :- ^ ProjectExecTransformer (12) @@ -145,7 +145,7 @@ Join condition: None Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) RegularHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt index 5f3ea7038ab0..b142ee1d8f33 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt @@ -1,44 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (46) +- == Final Plan == - VeloxColumnarToRowExec (30) - +- ^ SortExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- VeloxAppendBatches (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9), Statistics(X) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (31) + +- ^ SortExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ InputIteratorTransformer (21) + +- ShuffleQueryStage (19), Statistics(X) + +- ColumnarExchange (18) + +- VeloxAppendBatches (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- HashAggregate (42) - +- Exchange (41) - +- HashAggregate (40) - +- Project (39) - +- BroadcastHashJoin LeftSemi BuildRight (38) - :- Project (33) - : +- Filter (32) - : +- Scan parquet (31) - +- BroadcastExchange (37) - +- Project (36) - +- Filter (35) - +- Scan parquet (34) + Sort (45) + +- Exchange (44) + +- HashAggregate (43) + +- Exchange (42) + +- HashAggregate (41) + +- Project (40) + +- BroadcastHashJoin LeftSemi BuildRight (39) + :- Project (34) + : +- Filter (33) + : +- Scan parquet (32) + +- BroadcastExchange (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) (1) Scan parquet @@ -113,129 +114,133 @@ Input [2]: [o_orderpriority#X, count#X] Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(17) ColumnarExchange +(17) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(18) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(18) ShuffleQueryStage +(19) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(19) InputAdapter +(20) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(20) InputIteratorTransformer +(21) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(21) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) VeloxAppendBatches +(24) VeloxAppendBatches Input [2]: [o_orderpriority#X, order_count#X] Arguments: X -(24) ColumnarExchange +(25) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(28) SortExecTransformer +(29) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(29) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(30) VeloxColumnarToRowExec +(31) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(31) Scan parquet +(32) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(32) Filter +(33) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(33) Project +(34) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(34) Scan parquet +(35) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(35) Filter +(36) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(36) Project +(37) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(37) BroadcastExchange +(38) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(38) BroadcastHashJoin +(39) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(39) Project +(40) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(40) HashAggregate +(41) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(41) Exchange +(42) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) HashAggregate +(43) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(43) Exchange +(44) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(45) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(46) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt index d83f2507a100..c8a62c3aca1f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt @@ -1,92 +1,93 @@ == Physical Plan == -AdaptiveSparkPlan (101) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (66) - +- ^ SortExecTransformer (64) - +- ^ InputIteratorTransformer (63) - +- ShuffleQueryStage (61), Statistics(X) - +- ColumnarExchange (60) - +- VeloxAppendBatches (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45), Statistics(X) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (67) + +- ^ SortExecTransformer (65) + +- ^ InputIteratorTransformer (64) + +- ShuffleQueryStage (62), Statistics(X) + +- ColumnarExchange (61) + +- VeloxAppendBatches (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ InputIteratorTransformer (57) + +- ShuffleQueryStage (55), Statistics(X) + +- ColumnarExchange (54) + +- VeloxAppendBatches (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (100) - +- Exchange (99) - +- HashAggregate (98) - +- Exchange (97) - +- HashAggregate (96) - +- Project (95) - +- BroadcastHashJoin Inner BuildRight (94) - :- Project (89) - : +- BroadcastHashJoin Inner BuildRight (88) - : :- Project (84) - : : +- BroadcastHashJoin Inner BuildRight (83) - : : :- Project (79) - : : : +- BroadcastHashJoin Inner BuildRight (78) - : : : :- Project (74) - : : : : +- BroadcastHashJoin Inner BuildLeft (73) - : : : : :- BroadcastExchange (69) - : : : : : +- Filter (68) - : : : : : +- Scan parquet (67) - : : : : +- Project (72) - : : : : +- Filter (71) - : : : : +- Scan parquet (70) - : : : +- BroadcastExchange (77) - : : : +- Filter (76) - : : : +- Scan parquet (75) - : : +- BroadcastExchange (82) - : : +- Filter (81) - : : +- Scan parquet (80) - : +- BroadcastExchange (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (93) - +- Project (92) - +- Filter (91) - +- Scan parquet (90) + Sort (101) + +- Exchange (100) + +- HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Project (96) + +- BroadcastHashJoin Inner BuildRight (95) + :- Project (90) + : +- BroadcastHashJoin Inner BuildRight (89) + : :- Project (85) + : : +- BroadcastHashJoin Inner BuildRight (84) + : : :- Project (80) + : : : +- BroadcastHashJoin Inner BuildRight (79) + : : : :- Project (75) + : : : : +- BroadcastHashJoin Inner BuildLeft (74) + : : : : :- BroadcastExchange (70) + : : : : : +- Filter (69) + : : : : : +- Scan parquet (68) + : : : : +- Project (73) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (78) + : : : +- Filter (77) + : : : +- Scan parquet (76) + : : +- BroadcastExchange (83) + : : +- Filter (82) + : : +- Scan parquet (81) + : +- BroadcastExchange (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (94) + +- Project (93) + +- Filter (92) + +- Scan parquet (91) (1) Scan parquet @@ -313,225 +314,229 @@ Input [3]: [n_name#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(53) ColumnarExchange +(53) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(54) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(55) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(55) InputAdapter +(56) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(56) InputIteratorTransformer +(57) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(57) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(58) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(59) VeloxAppendBatches +(60) VeloxAppendBatches Input [2]: [n_name#X, revenue#X] Arguments: X -(60) ColumnarExchange +(61) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(62) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(62) InputAdapter +(63) InputAdapter Input [2]: [n_name#X, revenue#X] -(63) InputIteratorTransformer +(64) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(64) SortExecTransformer +(65) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(65) WholeStageCodegenTransformer (X) +(66) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(66) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(67) Scan parquet +(68) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(68) Filter +(69) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(69) BroadcastExchange +(70) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(70) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(72) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(74) Project +(75) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(75) Scan parquet +(76) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(76) Filter +(77) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(77) BroadcastExchange +(78) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(78) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(79) Project +(80) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(80) Scan parquet +(81) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(81) Filter +(82) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(82) BroadcastExchange +(83) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(83) BroadcastHashJoin +(84) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(84) Project +(85) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(85) Scan parquet +(86) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(87) BroadcastExchange +(88) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(89) Project +(90) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(90) Scan parquet +(91) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(91) Filter +(92) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(92) Project +(93) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(93) BroadcastExchange +(94) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(94) BroadcastHashJoin +(95) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(95) Project +(96) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(96) HashAggregate +(97) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(97) Exchange +(98) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) HashAggregate +(99) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(99) Exchange +(100) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Sort +(101) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(101) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt index 30e1b3fe013b..fa9c936a0ca1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt @@ -7,7 +7,7 @@ AdaptiveSparkPlan (20) +- ShuffleQueryStage (8), Statistics(X) +- ColumnarExchange (7) +- VeloxAppendBatches (6) - +- ^ RegularHashAggregateExecTransformer (4) + +- ^ FlushableHashAggregateExecTransformer (4) +- ^ ProjectExecTransformer (3) +- ^ NoopFilter (2) +- ^ Scan parquet (1) @@ -35,7 +35,7 @@ Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) RegularHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt index 24cd98650a17..f098b9ba837f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt @@ -1,86 +1,87 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (95) +- == Final Plan == - VeloxColumnarToRowExec (61) - +- ^ SortExecTransformer (59) - +- ^ InputIteratorTransformer (58) - +- ShuffleQueryStage (56), Statistics(X) - +- ColumnarExchange (55) - +- VeloxAppendBatches (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16), Statistics(X) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ NoopFilter (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25), Statistics(X) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ NoopFilter (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ NoopFilter (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40), Statistics(X) - +- ReusedExchange (39) + VeloxColumnarToRowExec (62) + +- ^ SortExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57), Statistics(X) + +- ColumnarExchange (56) + +- VeloxAppendBatches (55) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ NoopFilter (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ NoopFilter (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == - Sort (93) - +- Exchange (92) - +- HashAggregate (91) - +- Exchange (90) - +- HashAggregate (89) - +- Project (88) - +- BroadcastHashJoin Inner BuildRight (87) - :- Project (83) - : +- BroadcastHashJoin Inner BuildRight (82) - : :- Project (78) - : : +- BroadcastHashJoin Inner BuildRight (77) - : : :- Project (73) - : : : +- BroadcastHashJoin Inner BuildRight (72) - : : : :- Project (68) - : : : : +- BroadcastHashJoin Inner BuildLeft (67) - : : : : :- BroadcastExchange (64) - : : : : : +- Filter (63) - : : : : : +- Scan parquet (62) - : : : : +- Filter (66) - : : : : +- Scan parquet (65) - : : : +- BroadcastExchange (71) - : : : +- Filter (70) - : : : +- Scan parquet (69) - : : +- BroadcastExchange (76) - : : +- Filter (75) - : : +- Scan parquet (74) - : +- BroadcastExchange (81) - : +- Filter (80) - : +- Scan parquet (79) - +- BroadcastExchange (86) - +- Filter (85) - +- Scan parquet (84) + Sort (94) + +- Exchange (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- BroadcastHashJoin Inner BuildRight (88) + :- Project (84) + : +- BroadcastHashJoin Inner BuildRight (83) + : :- Project (79) + : : +- BroadcastHashJoin Inner BuildRight (78) + : : :- Project (74) + : : : +- BroadcastHashJoin Inner BuildRight (73) + : : : :- Project (69) + : : : : +- BroadcastHashJoin Inner BuildLeft (68) + : : : : :- BroadcastExchange (65) + : : : : : +- Filter (64) + : : : : : +- Scan parquet (63) + : : : : +- Filter (67) + : : : : +- Scan parquet (66) + : : : +- BroadcastExchange (72) + : : : +- Filter (71) + : : : +- Scan parquet (70) + : : +- BroadcastExchange (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- BroadcastExchange (82) + : +- Filter (81) + : +- Scan parquet (80) + +- BroadcastExchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -283,217 +284,221 @@ Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(48) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(49) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(50) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(53) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) VeloxAppendBatches +(55) VeloxAppendBatches Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(55) ColumnarExchange +(56) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(56) ShuffleQueryStage +(57) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(57) InputAdapter +(58) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) InputIteratorTransformer +(59) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(59) SortExecTransformer +(60) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(60) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(61) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(62) Scan parquet +(63) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(63) Filter +(64) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(64) BroadcastExchange +(65) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(65) Scan parquet +(66) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(67) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(68) Project +(69) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(69) Scan parquet +(70) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(71) BroadcastExchange +(72) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(73) Project +(74) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(75) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(78) Project +(79) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(79) Scan parquet +(80) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(80) Filter +(81) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(81) BroadcastExchange +(82) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(83) Project +(84) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(84) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(85) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(86) BroadcastExchange +(87) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(88) Project +(89) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(89) HashAggregate +(90) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(90) Exchange +(91) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(92) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(92) Exchange +(93) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Sort +(94) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(94) AdaptiveSparkPlan +(95) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt index 8185bc4a349e..becfc12a1b86 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt @@ -1,117 +1,118 @@ == Physical Plan == -AdaptiveSparkPlan (130) +AdaptiveSparkPlan (131) +- == Final Plan == - VeloxColumnarToRowExec (85) - +- ^ SortExecTransformer (83) - +- ^ InputIteratorTransformer (82) - +- ShuffleQueryStage (80), Statistics(X) - +- ColumnarExchange (79) - +- VeloxAppendBatches (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ NoopFilter (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26), Statistics(X) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ NoopFilter (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35), Statistics(X) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ NoopFilter (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44), Statistics(X) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ NoopFilter (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53), Statistics(X) - : +- ColumnarBroadcastExchange (52) - : +- ^ NoopFilter (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63), Statistics(X) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ NoopFilter (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (86) + +- ^ SortExecTransformer (84) + +- ^ InputIteratorTransformer (83) + +- ShuffleQueryStage (81), Statistics(X) + +- ColumnarExchange (80) + +- VeloxAppendBatches (79) + +- ^ ProjectExecTransformer (77) + +- ^ RegularHashAggregateExecTransformer (76) + +- ^ InputIteratorTransformer (75) + +- ShuffleQueryStage (73), Statistics(X) + +- ColumnarExchange (72) + +- VeloxAppendBatches (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ NoopFilter (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ NoopFilter (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ NoopFilter (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ NoopFilter (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ NoopFilter (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ NoopFilter (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (129) - +- Exchange (128) - +- HashAggregate (127) - +- Exchange (126) - +- HashAggregate (125) - +- Project (124) - +- BroadcastHashJoin Inner BuildRight (123) - :- Project (118) - : +- BroadcastHashJoin Inner BuildRight (117) - : :- Project (113) - : : +- BroadcastHashJoin Inner BuildRight (112) - : : :- Project (108) - : : : +- BroadcastHashJoin Inner BuildRight (107) - : : : :- Project (103) - : : : : +- BroadcastHashJoin Inner BuildRight (102) - : : : : :- Project (98) - : : : : : +- BroadcastHashJoin Inner BuildRight (97) - : : : : : :- Project (93) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (92) - : : : : : : :- BroadcastExchange (89) - : : : : : : : +- Project (88) - : : : : : : : +- Filter (87) - : : : : : : : +- Scan parquet (86) - : : : : : : +- Filter (91) - : : : : : : +- Scan parquet (90) - : : : : : +- BroadcastExchange (96) - : : : : : +- Filter (95) - : : : : : +- Scan parquet (94) - : : : : +- BroadcastExchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- BroadcastExchange (106) - : : : +- Filter (105) - : : : +- Scan parquet (104) - : : +- BroadcastExchange (111) - : : +- Filter (110) - : : +- Scan parquet (109) - : +- BroadcastExchange (116) - : +- Filter (115) - : +- Scan parquet (114) - +- BroadcastExchange (122) - +- Project (121) - +- Filter (120) - +- Scan parquet (119) + Sort (130) + +- Exchange (129) + +- HashAggregate (128) + +- Exchange (127) + +- HashAggregate (126) + +- Project (125) + +- BroadcastHashJoin Inner BuildRight (124) + :- Project (119) + : +- BroadcastHashJoin Inner BuildRight (118) + : :- Project (114) + : : +- BroadcastHashJoin Inner BuildRight (113) + : : :- Project (109) + : : : +- BroadcastHashJoin Inner BuildRight (108) + : : : :- Project (104) + : : : : +- BroadcastHashJoin Inner BuildRight (103) + : : : : :- Project (99) + : : : : : +- BroadcastHashJoin Inner BuildRight (98) + : : : : : :- Project (94) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (93) + : : : : : : :- BroadcastExchange (90) + : : : : : : : +- Project (89) + : : : : : : : +- Filter (88) + : : : : : : : +- Scan parquet (87) + : : : : : : +- Filter (92) + : : : : : : +- Scan parquet (91) + : : : : : +- BroadcastExchange (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- BroadcastExchange (102) + : : : : +- Filter (101) + : : : : +- Scan parquet (100) + : : : +- BroadcastExchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- BroadcastExchange (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- BroadcastExchange (117) + : +- Filter (116) + : +- Scan parquet (115) + +- BroadcastExchange (123) + +- Project (122) + +- Filter (121) + +- Scan parquet (120) (1) Scan parquet @@ -414,277 +415,281 @@ Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(71) ColumnarExchange +(71) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(72) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(72) ShuffleQueryStage +(73) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(73) InputAdapter +(74) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(74) InputIteratorTransformer +(75) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(75) RegularHashAggregateExecTransformer +(76) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(76) ProjectExecTransformer +(77) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(77) WholeStageCodegenTransformer (X) +(78) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) VeloxAppendBatches +(79) VeloxAppendBatches Input [2]: [o_year#X, mkt_share#X] Arguments: X -(79) ColumnarExchange +(80) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(80) ShuffleQueryStage +(81) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(81) InputAdapter +(82) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(82) InputIteratorTransformer +(83) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(83) SortExecTransformer +(84) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(84) WholeStageCodegenTransformer (X) +(85) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(85) VeloxColumnarToRowExec +(86) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(86) Scan parquet +(87) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(87) Filter +(88) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(88) Project +(89) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(89) BroadcastExchange +(90) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(90) Scan parquet +(91) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(91) Filter +(92) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(92) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(93) Project +(94) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(94) Scan parquet +(95) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(95) Filter +(96) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(96) BroadcastExchange +(97) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(97) BroadcastHashJoin +(98) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(98) Project +(99) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(99) Scan parquet +(100) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(100) Filter +(101) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(101) BroadcastExchange +(102) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(102) BroadcastHashJoin +(103) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(103) Project +(104) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(104) Scan parquet +(105) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(105) Filter +(106) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(106) BroadcastExchange +(107) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(107) BroadcastHashJoin +(108) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(108) Project +(109) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(109) Scan parquet +(110) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(110) Filter +(111) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(111) BroadcastExchange +(112) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(112) BroadcastHashJoin +(113) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(113) Project +(114) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(114) Scan parquet +(115) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(115) Filter +(116) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(116) BroadcastExchange +(117) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(117) BroadcastHashJoin +(118) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(118) Project +(119) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(119) Scan parquet +(120) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(120) Filter +(121) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(121) Project +(122) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(122) BroadcastExchange +(123) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(123) BroadcastHashJoin +(124) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(124) Project +(125) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(125) HashAggregate +(126) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(126) Exchange +(127) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) HashAggregate +(128) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(128) Exchange +(129) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Sort +(130) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(130) AdaptiveSparkPlan +(131) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt index 0d27f28fda4d..a486fc65677b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt @@ -1,90 +1,91 @@ == Physical Plan == -AdaptiveSparkPlan (99) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- VeloxAppendBatches (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61), Statistics(X) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (98) - +- Exchange (97) - +- HashAggregate (96) - +- Exchange (95) - +- HashAggregate (94) - +- Project (93) - +- BroadcastHashJoin Inner BuildRight (92) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (69) - : : : : : +- Project (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Filter (71) - : : : : +- Scan parquet (70) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (91) - +- Filter (90) - +- Scan parquet (89) + Sort (99) + +- Exchange (98) + +- HashAggregate (97) + +- Exchange (96) + +- HashAggregate (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (70) + : : : : : +- Project (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -307,221 +308,225 @@ Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(53) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) VeloxAppendBatches +(59) VeloxAppendBatches Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(59) ColumnarExchange +(60) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(61) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(61) InputAdapter +(62) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) InputIteratorTransformer +(63) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(63) SortExecTransformer +(64) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(65) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(67) Filter +(68) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(68) Project +(69) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(69) BroadcastExchange +(70) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(70) Scan parquet +(71) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(73) Project +(74) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(74) Scan parquet +(75) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(78) Project +(79) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(79) Scan parquet +(80) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(80) Filter +(81) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(81) BroadcastExchange +(82) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(83) Project +(84) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(84) Scan parquet +(85) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(86) BroadcastExchange +(87) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(88) Project +(89) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(89) Scan parquet +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(91) BroadcastExchange +(92) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(92) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(93) Project +(94) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(94) HashAggregate +(95) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(95) Exchange +(96) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) HashAggregate +(97) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(97) Exchange +(98) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Sort +(99) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(99) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt index 589019c4cdbf..545f2e7e086d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (29) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (28) - +- Exchange (27) - +- HashAggregate (26) - +- Exchange (25) - +- HashAggregate (24) - +- Project (23) - +- Filter (22) - +- Scan parquet (21) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -57,101 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) VeloxAppendBatches +(14) VeloxAppendBatches Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) ColumnarExchange +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(18) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(20) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(21) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(22) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(23) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(24) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(25) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(27) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(29) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt index f83547f843ca..9325f007789b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt @@ -10,7 +10,7 @@ AdaptiveSparkPlan (68) +- ColumnarExchange (36) +- VeloxAppendBatches (35) +- ^ ProjectExecTransformer (33) - +- ^ RegularHashAggregateExecTransformer (32) + +- ^ FlushableHashAggregateExecTransformer (32) +- ^ ProjectExecTransformer (31) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) :- ^ ProjectExecTransformer (22) @@ -198,7 +198,7 @@ Join condition: None Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(32) RegularHashAggregateExecTransformer +(32) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt index da2efbcb3134..c593374b9a00 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt @@ -1,56 +1,57 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- ^ SortExecTransformer (37) - +- ^ InputIteratorTransformer (36) - +- ShuffleQueryStage (34), Statistics(X) - +- ColumnarExchange (33) - +- VeloxAppendBatches (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7), Statistics(X) - : +- ColumnarBroadcastExchange (6) - : +- ^ NoopFilter (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17), Statistics(X) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ NoopFilter (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ FilterExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ NoopFilter (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ NoopFilter (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (58) - +- Exchange (57) - +- Filter (56) - +- HashAggregate (55) - +- Exchange (54) - +- HashAggregate (53) - +- Project (52) - +- BroadcastHashJoin Inner BuildRight (51) - :- Project (46) - : +- BroadcastHashJoin Inner BuildRight (45) - : :- Filter (41) - : : +- Scan parquet (40) - : +- BroadcastExchange (44) - : +- Filter (43) - : +- Scan parquet (42) - +- BroadcastExchange (50) - +- Project (49) - +- Filter (48) - +- Scan parquet (47) + Sort (59) + +- Exchange (58) + +- Filter (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- BroadcastHashJoin Inner BuildRight (52) + :- Project (47) + : +- BroadcastHashJoin Inner BuildRight (46) + : :- Filter (42) + : : +- Scan parquet (41) + : +- BroadcastExchange (45) + : +- Filter (44) + : +- Scan parquet (43) + +- BroadcastExchange (51) + +- Project (50) + +- Filter (49) + +- Scan parquet (48) (1) Scan parquet @@ -161,389 +162,398 @@ Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(25) ColumnarExchange +(25) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(26) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(27) InputAdapter +(28) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(28) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(29) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(30) FilterExecTransformer +(31) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) VeloxAppendBatches +(33) VeloxAppendBatches Input [2]: [ps_partkey#X, value#X] Arguments: X -(33) ColumnarExchange +(34) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(34) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(35) InputAdapter +(36) InputAdapter Input [2]: [ps_partkey#X, value#X] -(36) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(37) SortExecTransformer +(38) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(38) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(39) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(40) Scan parquet +(41) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(41) Filter +(42) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(42) Scan parquet +(43) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(43) Filter +(44) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(44) BroadcastExchange +(45) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(45) BroadcastHashJoin +(46) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(46) Project +(47) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(47) Scan parquet +(48) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(48) Filter +(49) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(49) Project +(50) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(50) BroadcastExchange +(51) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(51) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(52) Project +(53) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(53) HashAggregate +(54) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(54) Exchange +(55) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) HashAggregate +(56) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(56) Filter +(57) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(57) Exchange +(58) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) Sort +(59) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(59) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 30 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (100) +Subquery:1 Hosting operator id = 31 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (83) - +- ^ ProjectExecTransformer (81) - +- ^ RegularHashAggregateExecTransformer (80) - +- ^ InputIteratorTransformer (79) - +- ShuffleQueryStage (77), Statistics(X) - +- ColumnarExchange (76) - +- ^ FlushableHashAggregateExecTransformer (74) - +- ^ ProjectExecTransformer (73) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (72) - :- ^ ProjectExecTransformer (67) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - : :- ^ NoopFilter (61) - : : +- ^ Scan parquet (60) - : +- ^ InputIteratorTransformer (65) - : +- BroadcastQueryStage (63), Statistics(X) - : +- ReusedExchange (62) - +- ^ InputIteratorTransformer (71) - +- BroadcastQueryStage (69), Statistics(X) - +- ReusedExchange (68) + VeloxColumnarToRowExec (85) + +- ^ ProjectExecTransformer (83) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- VeloxAppendBatches (77) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (73) + :- ^ ProjectExecTransformer (68) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (67) + : :- ^ NoopFilter (62) + : : +- ^ Scan parquet (61) + : +- ^ InputIteratorTransformer (66) + : +- BroadcastQueryStage (64), Statistics(X) + : +- ReusedExchange (63) + +- ^ InputIteratorTransformer (72) + +- BroadcastQueryStage (70), Statistics(X) + +- ReusedExchange (69) +- == Initial Plan == - HashAggregate (99) - +- Exchange (98) - +- HashAggregate (97) - +- Project (96) - +- BroadcastHashJoin Inner BuildRight (95) - :- Project (90) - : +- BroadcastHashJoin Inner BuildRight (89) - : :- Filter (85) - : : +- Scan parquet (84) - : +- BroadcastExchange (88) - : +- Filter (87) - : +- Scan parquet (86) - +- BroadcastExchange (94) - +- Project (93) - +- Filter (92) - +- Scan parquet (91) - - -(60) Scan parquet + HashAggregate (101) + +- Exchange (100) + +- HashAggregate (99) + +- Project (98) + +- BroadcastHashJoin Inner BuildRight (97) + :- Project (92) + : +- BroadcastHashJoin Inner BuildRight (91) + : :- Filter (87) + : : +- Scan parquet (86) + : +- BroadcastExchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- BroadcastExchange (96) + +- Project (95) + +- Filter (94) + +- Scan parquet (93) + + +(61) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(61) NoopFilter +(62) NoopFilter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(62) ReusedExchange [Reuses operator id: 6] +(63) ReusedExchange [Reuses operator id: 6] Output [2]: [s_suppkey#X, s_nationkey#X] -(63) BroadcastQueryStage +(64) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(64) InputAdapter +(65) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(65) InputIteratorTransformer +(66) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(66) BroadcastHashJoinExecTransformer +(67) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(67) ProjectExecTransformer +(68) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(68) ReusedExchange [Reuses operator id: 16] +(69) ReusedExchange [Reuses operator id: 16] Output [1]: [n_nationkey#X] -(69) BroadcastQueryStage +(70) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(70) InputAdapter +(71) InputAdapter Input [1]: [n_nationkey#X] -(71) InputIteratorTransformer +(72) InputIteratorTransformer Input [1]: [n_nationkey#X] -(72) BroadcastHashJoinExecTransformer +(73) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(73) ProjectExecTransformer +(74) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(74) FlushableHashAggregateExecTransformer +(75) FlushableHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(75) WholeStageCodegenTransformer (X) +(76) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(76) ColumnarExchange +(77) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(78) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(77) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(78) InputAdapter +(80) InputAdapter Input [2]: [sum#X, isEmpty#X] -(79) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(80) RegularHashAggregateExecTransformer +(82) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(81) ProjectExecTransformer +(83) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(82) WholeStageCodegenTransformer (X) +(84) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(83) VeloxColumnarToRowExec +(85) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(84) Scan parquet +(86) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(85) Filter +(87) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(86) Scan parquet +(88) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(87) Filter +(89) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(88) BroadcastExchange +(90) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(89) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(90) Project +(92) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(91) Scan parquet +(93) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(92) Filter +(94) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(93) Project +(95) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(94) BroadcastExchange +(96) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(95) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(96) Project +(98) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(97) HashAggregate +(99) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(98) Exchange +(100) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(99) HashAggregate +(101) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(100) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt index 6006564bf2a8..93f3a4cf9ab7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt @@ -1,42 +1,43 @@ == Physical Plan == -AdaptiveSparkPlan (43) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- VeloxAppendBatches (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (42) - +- Exchange (41) - +- HashAggregate (40) - +- Exchange (39) - +- HashAggregate (38) - +- Project (37) - +- BroadcastHashJoin Inner BuildLeft (36) - :- BroadcastExchange (32) - : +- Filter (31) - : +- Scan parquet (30) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (43) + +- Exchange (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- BroadcastHashJoin Inner BuildLeft (37) + :- BroadcastExchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -108,126 +109,130 @@ Input [3]: [l_shipmode#X, sum#X, sum#X] Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(16) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(20) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) VeloxAppendBatches +(23) VeloxAppendBatches Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(23) ColumnarExchange +(24) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(27) SortExecTransformer +(28) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(30) Scan parquet +(31) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(32) BroadcastExchange +(33) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(33) Scan parquet +(34) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(34) Filter +(35) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(35) Project +(36) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(36) BroadcastHashJoin +(37) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(37) Project +(38) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(38) HashAggregate +(39) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(39) Exchange +(40) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) HashAggregate +(41) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(41) Exchange +(42) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Sort +(43) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(43) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt index 434a767c15a4..e44b10042401 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt @@ -1,50 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32), Statistics(X) - +- ColumnarExchange (31) - +- VeloxAppendBatches (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ NoopFilter (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ NoopFilter (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (52) - +- Exchange (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- BroadcastHashJoin LeftOuter BuildRight (43) - :- Scan parquet (38) - +- BroadcastExchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- BroadcastHashJoin LeftOuter BuildRight (45) + :- Scan parquet (40) + +- BroadcastExchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -111,179 +113,187 @@ Input [2]: [c_custkey#X, count#X] Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, count#X] +Arguments: X + +(16) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [c_custkey#X, count#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [c_custkey#X, count#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(22) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) VeloxAppendBatches +(32) VeloxAppendBatches Input [2]: [c_count#X, custdist#X] Arguments: X -(31) ColumnarExchange +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(33) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(34) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(35) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(36) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(37) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(38) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(39) Scan parquet +(41) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(42) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(43) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) BroadcastExchange +(44) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(43) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(44) Project +(46) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) Exchange +(48) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(48) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(49) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(51) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(53) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt index e4a7a29b15b0..8f5ae0f52e63 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt @@ -8,7 +8,7 @@ AdaptiveSparkPlan (35) +- ShuffleQueryStage (17), Statistics(X) +- ColumnarExchange (16) +- VeloxAppendBatches (15) - +- ^ RegularHashAggregateExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) @@ -87,7 +87,7 @@ Join condition: None Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(13) RegularHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt index f088b241bdd5..d616981e57d2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt @@ -1,43 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (43) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- AQEShuffleRead (26) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- VeloxAppendBatches (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) + VeloxColumnarToRowExec (28) + +- AQEShuffleRead (27) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (21) :- ^ InputIteratorTransformer (7) : +- BroadcastQueryStage (5), Statistics(X) : +- ColumnarBroadcastExchange (4) : +- ^ NoopFilter (2) : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- Project (39) - +- BroadcastHashJoin Inner BuildLeft (38) - :- BroadcastExchange (30) - : +- Filter (29) - : +- Scan parquet (28) - +- Filter (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (42) + +- Exchange (41) + +- Project (40) + +- BroadcastHashJoin Inner BuildLeft (39) + :- BroadcastExchange (31) + : +- Filter (30) + : +- Scan parquet (29) + +- Filter (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- Filter (33) + +- Scan parquet (32) (1) Scan parquet @@ -99,290 +100,299 @@ Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) VeloxAppendBatches +(24) VeloxAppendBatches Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(24) ColumnarExchange +(25) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) AQEShuffleRead +(27) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(27) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) Scan parquet +(29) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(29) Filter +(30) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(30) BroadcastExchange +(31) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(31) Scan parquet +(32) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(32) Filter +(33) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(33) Project +(34) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(34) HashAggregate +(35) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(35) Exchange +(36) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(37) Filter +(38) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(38) BroadcastHashJoin +(39) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(39) Project +(40) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(40) Exchange +(41) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(42) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(43) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 19 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (67) +Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ RegularHashAggregateExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ ProjectExecTransformer (47) - +- ^ FlushableHashAggregateExecTransformer (46) - +- ^ ProjectExecTransformer (45) - +- ^ NoopFilter (44) - +- ^ Scan parquet (43) + VeloxColumnarToRowExec (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ ProjectExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ InputIteratorTransformer (54) + +- ShuffleQueryStage (52), Statistics(X) + +- ColumnarExchange (51) + +- VeloxAppendBatches (50) + +- ^ ProjectExecTransformer (48) + +- ^ FlushableHashAggregateExecTransformer (47) + +- ^ ProjectExecTransformer (46) + +- ^ NoopFilter (45) + +- ^ Scan parquet (44) +- == Initial Plan == - HashAggregate (66) - +- HashAggregate (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (68) + +- HashAggregate (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- Filter (62) + +- Scan parquet (61) -(43) Scan parquet +(44) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(44) NoopFilter +(45) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(45) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) FlushableHashAggregateExecTransformer +(47) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(48) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(49) ColumnarExchange +(50) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(51) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(51) InputAdapter +(53) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(53) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(54) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(55) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(56) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(57) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(58) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(59) Scan parquet +(61) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(60) Filter +(62) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(61) Project +(63) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(63) Exchange +(65) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(65) HashAggregate +(67) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(66) HashAggregate +(68) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(67) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt index 3fff24e2b497..3b4da47fc25f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt @@ -1,54 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- VeloxAppendBatches (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ NoopFilter (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ NoopFilter (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (56) - +- Exchange (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- BroadcastHashJoin LeftAnti BuildRight (43) - : :- Filter (38) - : : +- Scan parquet (37) - : +- BroadcastExchange (42) - : +- Project (41) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Filter (45) - +- Scan parquet (44) + Sort (58) + +- Exchange (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Project (50) + +- BroadcastHashJoin Inner BuildRight (49) + :- BroadcastHashJoin LeftAnti BuildRight (45) + : :- Filter (40) + : : +- Scan parquet (39) + : +- BroadcastExchange (44) + : +- Project (43) + : +- Filter (42) + : +- Scan parquet (41) + +- BroadcastExchange (48) + +- Filter (47) + +- Scan parquet (46) (1) Scan parquet @@ -116,201 +118,209 @@ Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(16) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(25) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(26) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) VeloxAppendBatches +(31) VeloxAppendBatches Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(30) ColumnarExchange +(32) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(32) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(34) SortExecTransformer +(36) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(37) Scan parquet +(39) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(38) Filter +(40) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(39) Scan parquet +(41) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(40) Filter +(42) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(41) Project +(43) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(42) BroadcastExchange +(44) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(43) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: LeftAnti Join condition: None -(44) Scan parquet +(46) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(45) Filter +(47) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(46) BroadcastExchange +(48) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(47) BroadcastHashJoin +(49) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(48) Project +(50) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(49) HashAggregate +(51) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(50) Exchange +(52) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(52) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(53) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(56) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(55) Exchange +(57) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) Sort +(58) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(57) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/17.txt index e499293269fc..77733ad6f8a5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/17.txt @@ -8,7 +8,7 @@ AdaptiveSparkPlan (36) +- ShuffleQueryStage (9), Statistics(X) +- ColumnarExchange (8) +- VeloxAppendBatches (7) - +- ^ RegularHashAggregateExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (5) +- ^ InputIteratorTransformer (4) +- RowToVeloxColumnar (2) +- LocalTableScan (1) @@ -48,7 +48,7 @@ Input [1]: [l_extendedprice#X] (4) InputIteratorTransformer Input [1]: [l_extendedprice#X] -(5) RegularHashAggregateExecTransformer +(5) FlushableHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt index 3408aba7dcbc..0f5e82ec54c1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt @@ -9,7 +9,7 @@ AdaptiveSparkPlan (88) +- ColumnarExchange (48) +- VeloxAppendBatches (47) +- ^ ProjectExecTransformer (45) - +- ^ RegularHashAggregateExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (44) +- ^ ProjectExecTransformer (43) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) :- ^ ProjectExecTransformer (29) @@ -33,7 +33,7 @@ AdaptiveSparkPlan (88) : +- ColumnarExchange (15) : +- VeloxAppendBatches (14) : +- ^ ProjectExecTransformer (12) - : +- ^ RegularHashAggregateExecTransformer (11) + : +- ^ FlushableHashAggregateExecTransformer (11) : +- ^ Scan parquet (10) +- ^ InputIteratorTransformer (41) +- BroadcastQueryStage (39), Statistics(X) @@ -125,7 +125,7 @@ Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) RegularHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] @@ -265,7 +265,7 @@ Join condition: None Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(44) RegularHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt index 7fa7f7108a17..3bafdb994153 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt @@ -7,7 +7,7 @@ AdaptiveSparkPlan (34) +- ShuffleQueryStage (17), Statistics(X) +- ColumnarExchange (16) +- VeloxAppendBatches (15) - +- ^ RegularHashAggregateExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) @@ -86,7 +86,7 @@ Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) RegularHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt index a4be44f50e60..945cc79de1ac 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt @@ -1,22 +1,22 @@ == Physical Plan == -AdaptiveSparkPlan (97) +AdaptiveSparkPlan (98) +- == Final Plan == - VeloxColumnarToRowExec (61) - +- AQEShuffleRead (60) - +- ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- VeloxAppendBatches (57) - +- ^ ProjectExecTransformer (55) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (54) - :- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (44) + VeloxColumnarToRowExec (62) + +- AQEShuffleRead (61) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ ProjectExecTransformer (56) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (55) + :- ^ ProjectExecTransformer (46) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (45) : :- ^ NoopFilter (2) : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (43) - : +- BroadcastQueryStage (41), Statistics(X) - : +- ColumnarBroadcastExchange (40) - : +- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (37) + : +- ^ InputIteratorTransformer (44) + : +- BroadcastQueryStage (42), Statistics(X) + : +- ColumnarBroadcastExchange (41) + : +- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (38) : :- ^ InputIteratorTransformer (18) : : +- BroadcastQueryStage (16), Statistics(X) : : +- ColumnarBroadcastExchange (15) @@ -29,63 +29,64 @@ AdaptiveSparkPlan (97) : : +- ^ ProjectExecTransformer (7) : : +- ^ NoopFilter (6) : : +- ^ Scan parquet (5) - : +- ^ FilterExecTransformer (36) - : +- ^ ProjectExecTransformer (35) - : +- ^ RegularHashAggregateExecTransformer (34) - : +- ^ InputIteratorTransformer (33) - : +- ShuffleQueryStage (31), Statistics(X) - : +- ColumnarExchange (30) - : +- ^ ProjectExecTransformer (28) - : +- ^ FlushableHashAggregateExecTransformer (27) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ ProjectExecTransformer (21) - : : +- ^ NoopFilter (20) - : : +- ^ Scan parquet (19) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23), Statistics(X) - : +- ReusedExchange (22) - +- ^ InputIteratorTransformer (53) - +- BroadcastQueryStage (51), Statistics(X) - +- ColumnarBroadcastExchange (50) - +- ^ ProjectExecTransformer (48) - +- ^ NoopFilter (47) - +- ^ Scan parquet (46) + : +- ^ FilterExecTransformer (37) + : +- ^ ProjectExecTransformer (36) + : +- ^ RegularHashAggregateExecTransformer (35) + : +- ^ InputIteratorTransformer (34) + : +- ShuffleQueryStage (32), Statistics(X) + : +- ColumnarExchange (31) + : +- VeloxAppendBatches (30) + : +- ^ ProjectExecTransformer (28) + : +- ^ FlushableHashAggregateExecTransformer (27) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) + : :- ^ ProjectExecTransformer (21) + : : +- ^ NoopFilter (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (25) + : +- BroadcastQueryStage (23), Statistics(X) + : +- ReusedExchange (22) + +- ^ InputIteratorTransformer (54) + +- BroadcastQueryStage (52), Statistics(X) + +- ColumnarBroadcastExchange (51) + +- ^ ProjectExecTransformer (49) + +- ^ NoopFilter (48) + +- ^ Scan parquet (47) +- == Initial Plan == - Sort (96) - +- Exchange (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (88) - : +- BroadcastHashJoin LeftSemi BuildRight (87) - : :- Filter (63) - : : +- Scan parquet (62) - : +- BroadcastExchange (86) - : +- Project (85) - : +- BroadcastHashJoin Inner BuildLeft (84) - : :- BroadcastExchange (71) - : : +- BroadcastHashJoin LeftSemi BuildRight (70) - : : :- Filter (65) - : : : +- Scan parquet (64) - : : +- BroadcastExchange (69) - : : +- Project (68) - : : +- Filter (67) - : : +- Scan parquet (66) - : +- Filter (83) - : +- HashAggregate (82) - : +- Exchange (81) - : +- HashAggregate (80) - : +- BroadcastHashJoin LeftSemi BuildRight (79) - : :- Project (74) - : : +- Filter (73) - : : +- Scan parquet (72) - : +- BroadcastExchange (78) - : +- Project (77) - : +- Filter (76) - : +- Scan parquet (75) - +- BroadcastExchange (92) - +- Project (91) - +- Filter (90) - +- Scan parquet (89) + Sort (97) + +- Exchange (96) + +- Project (95) + +- BroadcastHashJoin Inner BuildRight (94) + :- Project (89) + : +- BroadcastHashJoin LeftSemi BuildRight (88) + : :- Filter (64) + : : +- Scan parquet (63) + : +- BroadcastExchange (87) + : +- Project (86) + : +- BroadcastHashJoin Inner BuildLeft (85) + : :- BroadcastExchange (72) + : : +- BroadcastHashJoin LeftSemi BuildRight (71) + : : :- Filter (66) + : : : +- Scan parquet (65) + : : +- BroadcastExchange (70) + : : +- Project (69) + : : +- Filter (68) + : : +- Scan parquet (67) + : +- Filter (84) + : +- HashAggregate (83) + : +- Exchange (82) + : +- HashAggregate (81) + : +- BroadcastHashJoin LeftSemi BuildRight (80) + : :- Project (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- BroadcastExchange (79) + : +- Project (78) + : +- Filter (77) + : +- Scan parquet (76) + +- BroadcastExchange (93) + +- Project (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -216,313 +217,317 @@ Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(30) ColumnarExchange +(30) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(31) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(32) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(32) InputAdapter +(33) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(33) InputIteratorTransformer +(34) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(35) ProjectExecTransformer +(36) ProjectExecTransformer Output [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(36) FilterExecTransformer +(37) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(37) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(38) ProjectExecTransformer +(39) ProjectExecTransformer Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(39) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] Arguments: false -(40) ColumnarBroadcastExchange +(41) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(41) BroadcastQueryStage +(42) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(42) InputAdapter +(43) InputAdapter Input [1]: [ps_suppkey#X] -(43) InputIteratorTransformer +(44) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(44) BroadcastHashJoinExecTransformer +(45) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(45) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(46) Scan parquet +(47) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(47) NoopFilter +(48) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(48) ProjectExecTransformer +(49) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(50) ColumnarBroadcastExchange +(51) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(51) BroadcastQueryStage +(52) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(52) InputAdapter +(53) InputAdapter Input [1]: [n_nationkey#X] -(53) InputIteratorTransformer +(54) InputIteratorTransformer Input [1]: [n_nationkey#X] -(54) BroadcastHashJoinExecTransformer +(55) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(55) ProjectExecTransformer +(56) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(56) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(57) VeloxAppendBatches +(58) VeloxAppendBatches Input [2]: [s_name#X, s_address#X] Arguments: X -(58) ColumnarExchange +(59) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(60) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(60) AQEShuffleRead +(61) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(61) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(62) Scan parquet +(63) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(63) Filter +(64) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(64) Scan parquet +(65) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(65) Filter +(66) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(66) Scan parquet +(67) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(67) Filter +(68) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(68) Project +(69) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(69) BroadcastExchange +(70) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(70) BroadcastHashJoin +(71) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(71) BroadcastExchange +(72) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(72) Scan parquet +(73) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(73) Filter +(74) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(74) Project +(75) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(75) Scan parquet +(76) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(76) Filter +(77) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(77) Project +(78) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(78) BroadcastExchange +(79) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(79) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(80) HashAggregate +(81) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(81) Exchange +(82) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) HashAggregate +(83) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(83) Filter +(84) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(84) BroadcastHashJoin +(85) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(85) Project +(86) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(86) BroadcastExchange +(87) BroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(88) Project +(89) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(89) Scan parquet +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(91) Project +(92) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(92) BroadcastExchange +(93) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(93) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(94) Project +(95) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(95) Exchange +(96) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) Sort +(97) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(97) AdaptiveSparkPlan +(98) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt index 89f395ed56d5..a7d9f42063a0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt @@ -8,7 +8,7 @@ AdaptiveSparkPlan (92) +- ColumnarExchange (53) +- VeloxAppendBatches (52) +- ^ ProjectExecTransformer (50) - +- ^ RegularHashAggregateExecTransformer (49) + +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) :- ^ ProjectExecTransformer (38) @@ -291,7 +291,7 @@ Join condition: None Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(49) RegularHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt index 388bd109476a..0d9139bcd990 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt @@ -1,38 +1,39 @@ == Physical Plan == -AdaptiveSparkPlan (39) +AdaptiveSparkPlan (40) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- ^ SortExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- VeloxAppendBatches (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (28) + +- ^ SortExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- VeloxAppendBatches (21) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (38) - +- Exchange (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- BroadcastHashJoin LeftAnti BuildRight (32) - :- Filter (29) - : +- Scan parquet (28) - +- BroadcastExchange (31) - +- Scan parquet (30) + Sort (39) + +- Exchange (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- BroadcastHashJoin LeftAnti BuildRight (33) + :- Filter (30) + : +- Scan parquet (29) + +- BroadcastExchange (32) + +- Scan parquet (31) (1) Scan parquet @@ -95,231 +96,240 @@ Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) VeloxAppendBatches +(21) VeloxAppendBatches Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(21) ColumnarExchange +(22) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(23) InputAdapter +(24) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(25) SortExecTransformer +(26) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(26) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(28) Scan parquet +(29) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(29) Filter +(30) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(30) Scan parquet +(31) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(31) BroadcastExchange +(32) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(32) BroadcastHashJoin +(33) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(33) Project +(34) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(34) HashAggregate +(35) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(35) Exchange +(36) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(37) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(37) Exchange +(38) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Sort +(39) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(39) AdaptiveSparkPlan +(40) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (51) - +- ^ RegularHashAggregateExecTransformer (49) - +- ^ InputIteratorTransformer (48) - +- ShuffleQueryStage (46), Statistics(X) - +- ColumnarExchange (45) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (53) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ShuffleQueryStage (48), Statistics(X) + +- ColumnarExchange (47) + +- VeloxAppendBatches (46) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ NoopFilter (42) + +- ^ Scan parquet (41) +- == Initial Plan == - HashAggregate (57) - +- Exchange (56) - +- HashAggregate (55) - +- Project (54) - +- Filter (53) - +- Scan parquet (52) + HashAggregate (59) + +- Exchange (58) + +- HashAggregate (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) -(40) Scan parquet +(41) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(41) NoopFilter +(42) NoopFilter Input [2]: [c_phone#X, c_acctbal#X] Arguments: [c_phone#X, c_acctbal#X] -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(43) FlushableHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(44) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(45) ColumnarExchange +(46) VeloxAppendBatches +Input [2]: [sum#X, count#X] +Arguments: X + +(47) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(46) ShuffleQueryStage +(48) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(47) InputAdapter +(49) InputAdapter Input [2]: [sum#X, count#X] -(48) InputIteratorTransformer +(50) InputIteratorTransformer Input [2]: [sum#X, count#X] -(49) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(50) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(51) VeloxColumnarToRowExec +(53) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(52) Scan parquet +(54) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(53) Filter +(55) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(54) Project +(56) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(55) HashAggregate +(57) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(56) Exchange +(58) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(57) HashAggregate +(59) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(58) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt index c157058fc2ef..988bdc6c26ca 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt @@ -10,7 +10,7 @@ AdaptiveSparkPlan (54) +- ColumnarExchange (27) +- VeloxAppendBatches (26) +- ^ ProjectExecTransformer (24) - +- ^ RegularHashAggregateExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (23) +- ^ ProjectExecTransformer (22) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) :- ^ ProjectExecTransformer (12) @@ -147,7 +147,7 @@ Join condition: None Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) RegularHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt index 5e3f77bb06da..0b73608fbe4b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt @@ -1,44 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (46) +- == Final Plan == - VeloxColumnarToRowExec (30) - +- ^ SortExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- VeloxAppendBatches (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9), Statistics(X) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (31) + +- ^ SortExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ InputIteratorTransformer (21) + +- ShuffleQueryStage (19), Statistics(X) + +- ColumnarExchange (18) + +- VeloxAppendBatches (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- HashAggregate (42) - +- Exchange (41) - +- HashAggregate (40) - +- Project (39) - +- BroadcastHashJoin LeftSemi BuildRight (38) - :- Project (33) - : +- Filter (32) - : +- Scan parquet (31) - +- BroadcastExchange (37) - +- Project (36) - +- Filter (35) - +- Scan parquet (34) + Sort (45) + +- Exchange (44) + +- HashAggregate (43) + +- Exchange (42) + +- HashAggregate (41) + +- Project (40) + +- BroadcastHashJoin LeftSemi BuildRight (39) + :- Project (34) + : +- Filter (33) + : +- Scan parquet (32) + +- BroadcastExchange (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) (1) Scan parquet @@ -114,130 +115,134 @@ Input [2]: [o_orderpriority#X, count#X] Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(17) ColumnarExchange +(17) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(18) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(18) ShuffleQueryStage +(19) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(19) InputAdapter +(20) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(20) InputIteratorTransformer +(21) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(21) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) VeloxAppendBatches +(24) VeloxAppendBatches Input [2]: [o_orderpriority#X, order_count#X] Arguments: X -(24) ColumnarExchange +(25) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(28) SortExecTransformer +(29) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(29) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(30) VeloxColumnarToRowExec +(31) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(31) Scan parquet +(32) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(32) Filter +(33) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(33) Project +(34) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(34) Scan parquet +(35) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(35) Filter +(36) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(36) Project +(37) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(37) BroadcastExchange +(38) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(38) BroadcastHashJoin +(39) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(39) Project +(40) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(40) HashAggregate +(41) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(41) Exchange +(42) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) HashAggregate +(43) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(43) Exchange +(44) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(45) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(46) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt index 0f5d19f82450..8891188e8a08 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt @@ -1,92 +1,93 @@ == Physical Plan == -AdaptiveSparkPlan (101) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (66) - +- ^ SortExecTransformer (64) - +- ^ InputIteratorTransformer (63) - +- ShuffleQueryStage (61), Statistics(X) - +- ColumnarExchange (60) - +- VeloxAppendBatches (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45), Statistics(X) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (67) + +- ^ SortExecTransformer (65) + +- ^ InputIteratorTransformer (64) + +- ShuffleQueryStage (62), Statistics(X) + +- ColumnarExchange (61) + +- VeloxAppendBatches (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ InputIteratorTransformer (57) + +- ShuffleQueryStage (55), Statistics(X) + +- ColumnarExchange (54) + +- VeloxAppendBatches (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (100) - +- Exchange (99) - +- HashAggregate (98) - +- Exchange (97) - +- HashAggregate (96) - +- Project (95) - +- BroadcastHashJoin Inner BuildRight (94) - :- Project (89) - : +- BroadcastHashJoin Inner BuildRight (88) - : :- Project (84) - : : +- BroadcastHashJoin Inner BuildRight (83) - : : :- Project (79) - : : : +- BroadcastHashJoin Inner BuildRight (78) - : : : :- Project (74) - : : : : +- BroadcastHashJoin Inner BuildLeft (73) - : : : : :- BroadcastExchange (69) - : : : : : +- Filter (68) - : : : : : +- Scan parquet (67) - : : : : +- Project (72) - : : : : +- Filter (71) - : : : : +- Scan parquet (70) - : : : +- BroadcastExchange (77) - : : : +- Filter (76) - : : : +- Scan parquet (75) - : : +- BroadcastExchange (82) - : : +- Filter (81) - : : +- Scan parquet (80) - : +- BroadcastExchange (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (93) - +- Project (92) - +- Filter (91) - +- Scan parquet (90) + Sort (101) + +- Exchange (100) + +- HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Project (96) + +- BroadcastHashJoin Inner BuildRight (95) + :- Project (90) + : +- BroadcastHashJoin Inner BuildRight (89) + : :- Project (85) + : : +- BroadcastHashJoin Inner BuildRight (84) + : : :- Project (80) + : : : +- BroadcastHashJoin Inner BuildRight (79) + : : : :- Project (75) + : : : : +- BroadcastHashJoin Inner BuildLeft (74) + : : : : :- BroadcastExchange (70) + : : : : : +- Filter (69) + : : : : : +- Scan parquet (68) + : : : : +- Project (73) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (78) + : : : +- Filter (77) + : : : +- Scan parquet (76) + : : +- BroadcastExchange (83) + : : +- Filter (82) + : : +- Scan parquet (81) + : +- BroadcastExchange (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (94) + +- Project (93) + +- Filter (92) + +- Scan parquet (91) (1) Scan parquet @@ -318,230 +319,234 @@ Input [3]: [n_name#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(53) ColumnarExchange +(53) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(54) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(55) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(55) InputAdapter +(56) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(56) InputIteratorTransformer +(57) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(57) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(58) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(59) VeloxAppendBatches +(60) VeloxAppendBatches Input [2]: [n_name#X, revenue#X] Arguments: X -(60) ColumnarExchange +(61) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(62) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(62) InputAdapter +(63) InputAdapter Input [2]: [n_name#X, revenue#X] -(63) InputIteratorTransformer +(64) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(64) SortExecTransformer +(65) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(65) WholeStageCodegenTransformer (X) +(66) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(66) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(67) Scan parquet +(68) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(68) Filter +(69) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(69) BroadcastExchange +(70) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(70) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(72) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(74) Project +(75) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(75) Scan parquet +(76) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(76) Filter +(77) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(77) BroadcastExchange +(78) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(78) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(79) Project +(80) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(80) Scan parquet +(81) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(81) Filter +(82) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(82) BroadcastExchange +(83) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(83) BroadcastHashJoin +(84) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(84) Project +(85) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(85) Scan parquet +(86) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(87) BroadcastExchange +(88) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(89) Project +(90) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(90) Scan parquet +(91) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(91) Filter +(92) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(92) Project +(93) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(93) BroadcastExchange +(94) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(94) BroadcastHashJoin +(95) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(95) Project +(96) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(96) HashAggregate +(97) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(97) Exchange +(98) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) HashAggregate +(99) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(99) Exchange +(100) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Sort +(101) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(101) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt index fbcc75a247d5..64624c791f72 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt @@ -7,7 +7,7 @@ AdaptiveSparkPlan (20) +- ShuffleQueryStage (8), Statistics(X) +- ColumnarExchange (7) +- VeloxAppendBatches (6) - +- ^ RegularHashAggregateExecTransformer (4) + +- ^ FlushableHashAggregateExecTransformer (4) +- ^ ProjectExecTransformer (3) +- ^ NoopFilter (2) +- ^ Scan parquet (1) @@ -35,7 +35,7 @@ Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * l_discount#X) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) RegularHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt index 7a9949d5358c..ae1ab637f805 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt @@ -1,86 +1,87 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (95) +- == Final Plan == - VeloxColumnarToRowExec (61) - +- ^ SortExecTransformer (59) - +- ^ InputIteratorTransformer (58) - +- ShuffleQueryStage (56), Statistics(X) - +- ColumnarExchange (55) - +- VeloxAppendBatches (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16), Statistics(X) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ NoopFilter (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25), Statistics(X) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ NoopFilter (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ NoopFilter (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40), Statistics(X) - +- ReusedExchange (39) + VeloxColumnarToRowExec (62) + +- ^ SortExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57), Statistics(X) + +- ColumnarExchange (56) + +- VeloxAppendBatches (55) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ NoopFilter (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ NoopFilter (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == - Sort (93) - +- Exchange (92) - +- HashAggregate (91) - +- Exchange (90) - +- HashAggregate (89) - +- Project (88) - +- BroadcastHashJoin Inner BuildRight (87) - :- Project (83) - : +- BroadcastHashJoin Inner BuildRight (82) - : :- Project (78) - : : +- BroadcastHashJoin Inner BuildRight (77) - : : :- Project (73) - : : : +- BroadcastHashJoin Inner BuildRight (72) - : : : :- Project (68) - : : : : +- BroadcastHashJoin Inner BuildLeft (67) - : : : : :- BroadcastExchange (64) - : : : : : +- Filter (63) - : : : : : +- Scan parquet (62) - : : : : +- Filter (66) - : : : : +- Scan parquet (65) - : : : +- BroadcastExchange (71) - : : : +- Filter (70) - : : : +- Scan parquet (69) - : : +- BroadcastExchange (76) - : : +- Filter (75) - : : +- Scan parquet (74) - : +- BroadcastExchange (81) - : +- Filter (80) - : +- Scan parquet (79) - +- BroadcastExchange (86) - +- Filter (85) - +- Scan parquet (84) + Sort (94) + +- Exchange (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- BroadcastHashJoin Inner BuildRight (88) + :- Project (84) + : +- BroadcastHashJoin Inner BuildRight (83) + : :- Project (79) + : : +- BroadcastHashJoin Inner BuildRight (78) + : : :- Project (74) + : : : +- BroadcastHashJoin Inner BuildRight (73) + : : : :- Project (69) + : : : : +- BroadcastHashJoin Inner BuildLeft (68) + : : : : :- BroadcastExchange (65) + : : : : : +- Filter (64) + : : : : : +- Scan parquet (63) + : : : : +- Filter (67) + : : : : +- Scan parquet (66) + : : : +- BroadcastExchange (72) + : : : +- Filter (71) + : : : +- Scan parquet (70) + : : +- BroadcastExchange (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- BroadcastExchange (82) + : +- Filter (81) + : +- Scan parquet (80) + +- BroadcastExchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -288,222 +289,226 @@ Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(48) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(49) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(50) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(53) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) VeloxAppendBatches +(55) VeloxAppendBatches Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(55) ColumnarExchange +(56) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(56) ShuffleQueryStage +(57) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(57) InputAdapter +(58) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) InputIteratorTransformer +(59) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(59) SortExecTransformer +(60) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(60) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(61) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(62) Scan parquet +(63) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(63) Filter +(64) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(64) BroadcastExchange +(65) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(65) Scan parquet +(66) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(67) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(68) Project +(69) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(69) Scan parquet +(70) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(71) BroadcastExchange +(72) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(73) Project +(74) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(75) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(78) Project +(79) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(79) Scan parquet +(80) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(80) Filter +(81) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(81) BroadcastExchange +(82) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(83) Project +(84) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(84) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(85) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(86) BroadcastExchange +(87) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(88) Project +(89) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(89) HashAggregate +(90) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(90) Exchange +(91) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(92) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(92) Exchange +(93) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Sort +(94) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(94) AdaptiveSparkPlan +(95) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt index e5fd2360e758..4bbf7967744c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt @@ -1,117 +1,118 @@ == Physical Plan == -AdaptiveSparkPlan (130) +AdaptiveSparkPlan (131) +- == Final Plan == - VeloxColumnarToRowExec (85) - +- ^ SortExecTransformer (83) - +- ^ InputIteratorTransformer (82) - +- ShuffleQueryStage (80), Statistics(X) - +- ColumnarExchange (79) - +- VeloxAppendBatches (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ NoopFilter (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26), Statistics(X) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ NoopFilter (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35), Statistics(X) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ NoopFilter (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44), Statistics(X) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ NoopFilter (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53), Statistics(X) - : +- ColumnarBroadcastExchange (52) - : +- ^ NoopFilter (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63), Statistics(X) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ NoopFilter (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (86) + +- ^ SortExecTransformer (84) + +- ^ InputIteratorTransformer (83) + +- ShuffleQueryStage (81), Statistics(X) + +- ColumnarExchange (80) + +- VeloxAppendBatches (79) + +- ^ ProjectExecTransformer (77) + +- ^ RegularHashAggregateExecTransformer (76) + +- ^ InputIteratorTransformer (75) + +- ShuffleQueryStage (73), Statistics(X) + +- ColumnarExchange (72) + +- VeloxAppendBatches (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ NoopFilter (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ NoopFilter (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ NoopFilter (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ NoopFilter (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ NoopFilter (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ NoopFilter (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (129) - +- Exchange (128) - +- HashAggregate (127) - +- Exchange (126) - +- HashAggregate (125) - +- Project (124) - +- BroadcastHashJoin Inner BuildRight (123) - :- Project (118) - : +- BroadcastHashJoin Inner BuildRight (117) - : :- Project (113) - : : +- BroadcastHashJoin Inner BuildRight (112) - : : :- Project (108) - : : : +- BroadcastHashJoin Inner BuildRight (107) - : : : :- Project (103) - : : : : +- BroadcastHashJoin Inner BuildRight (102) - : : : : :- Project (98) - : : : : : +- BroadcastHashJoin Inner BuildRight (97) - : : : : : :- Project (93) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (92) - : : : : : : :- BroadcastExchange (89) - : : : : : : : +- Project (88) - : : : : : : : +- Filter (87) - : : : : : : : +- Scan parquet (86) - : : : : : : +- Filter (91) - : : : : : : +- Scan parquet (90) - : : : : : +- BroadcastExchange (96) - : : : : : +- Filter (95) - : : : : : +- Scan parquet (94) - : : : : +- BroadcastExchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- BroadcastExchange (106) - : : : +- Filter (105) - : : : +- Scan parquet (104) - : : +- BroadcastExchange (111) - : : +- Filter (110) - : : +- Scan parquet (109) - : +- BroadcastExchange (116) - : +- Filter (115) - : +- Scan parquet (114) - +- BroadcastExchange (122) - +- Project (121) - +- Filter (120) - +- Scan parquet (119) + Sort (130) + +- Exchange (129) + +- HashAggregate (128) + +- Exchange (127) + +- HashAggregate (126) + +- Project (125) + +- BroadcastHashJoin Inner BuildRight (124) + :- Project (119) + : +- BroadcastHashJoin Inner BuildRight (118) + : :- Project (114) + : : +- BroadcastHashJoin Inner BuildRight (113) + : : :- Project (109) + : : : +- BroadcastHashJoin Inner BuildRight (108) + : : : :- Project (104) + : : : : +- BroadcastHashJoin Inner BuildRight (103) + : : : : :- Project (99) + : : : : : +- BroadcastHashJoin Inner BuildRight (98) + : : : : : :- Project (94) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (93) + : : : : : : :- BroadcastExchange (90) + : : : : : : : +- Project (89) + : : : : : : : +- Filter (88) + : : : : : : : +- Scan parquet (87) + : : : : : : +- Filter (92) + : : : : : : +- Scan parquet (91) + : : : : : +- BroadcastExchange (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- BroadcastExchange (102) + : : : : +- Filter (101) + : : : : +- Scan parquet (100) + : : : +- BroadcastExchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- BroadcastExchange (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- BroadcastExchange (117) + : +- Filter (116) + : +- Scan parquet (115) + +- BroadcastExchange (123) + +- Project (122) + +- Filter (121) + +- Scan parquet (120) (1) Scan parquet @@ -421,284 +422,288 @@ Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(71) ColumnarExchange +(71) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(72) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(72) ShuffleQueryStage +(73) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(73) InputAdapter +(74) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(74) InputIteratorTransformer +(75) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(75) RegularHashAggregateExecTransformer +(76) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(76) ProjectExecTransformer +(77) ProjectExecTransformer Output [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(77) WholeStageCodegenTransformer (X) +(78) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) VeloxAppendBatches +(79) VeloxAppendBatches Input [2]: [o_year#X, mkt_share#X] Arguments: X -(79) ColumnarExchange +(80) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(80) ShuffleQueryStage +(81) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(81) InputAdapter +(82) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(82) InputIteratorTransformer +(83) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(83) SortExecTransformer +(84) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(84) WholeStageCodegenTransformer (X) +(85) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(85) VeloxColumnarToRowExec +(86) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(86) Scan parquet +(87) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(87) Filter +(88) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(88) Project +(89) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(89) BroadcastExchange +(90) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(90) Scan parquet +(91) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(91) Filter +(92) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(92) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(93) Project +(94) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(94) Scan parquet +(95) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(95) Filter +(96) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(96) BroadcastExchange +(97) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(97) BroadcastHashJoin +(98) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(98) Project +(99) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(99) Scan parquet +(100) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(100) Filter +(101) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(101) BroadcastExchange +(102) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(102) BroadcastHashJoin +(103) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(103) Project +(104) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(104) Scan parquet +(105) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(105) Filter +(106) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(106) BroadcastExchange +(107) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(107) BroadcastHashJoin +(108) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(108) Project +(109) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(109) Scan parquet +(110) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(110) Filter +(111) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(111) BroadcastExchange +(112) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(112) BroadcastHashJoin +(113) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(113) Project +(114) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(114) Scan parquet +(115) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(115) Filter +(116) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(116) BroadcastExchange +(117) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(117) BroadcastHashJoin +(118) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(118) Project +(119) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(119) Scan parquet +(120) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(120) Filter +(121) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(121) Project +(122) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(122) BroadcastExchange +(123) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(123) BroadcastHashJoin +(124) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(124) Project +(125) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(125) HashAggregate +(126) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(126) Exchange +(127) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) HashAggregate +(128) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(128) Exchange +(129) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Sort +(130) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(130) AdaptiveSparkPlan +(131) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt index 4538df9c485a..301c001eb793 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt @@ -1,90 +1,91 @@ == Physical Plan == -AdaptiveSparkPlan (99) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- VeloxAppendBatches (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61), Statistics(X) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (98) - +- Exchange (97) - +- HashAggregate (96) - +- Exchange (95) - +- HashAggregate (94) - +- Project (93) - +- BroadcastHashJoin Inner BuildRight (92) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (69) - : : : : : +- Project (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Filter (71) - : : : : +- Scan parquet (70) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (91) - +- Filter (90) - +- Scan parquet (89) + Sort (99) + +- Exchange (98) + +- HashAggregate (97) + +- Exchange (96) + +- HashAggregate (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (70) + : : : : : +- Project (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -312,226 +313,230 @@ Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(53) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) VeloxAppendBatches +(59) VeloxAppendBatches Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(59) ColumnarExchange +(60) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(61) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(61) InputAdapter +(62) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) InputIteratorTransformer +(63) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(63) SortExecTransformer +(64) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(65) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(67) Filter +(68) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(68) Project +(69) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(69) BroadcastExchange +(70) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(70) Scan parquet +(71) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(73) Project +(74) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(74) Scan parquet +(75) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(78) Project +(79) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(79) Scan parquet +(80) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(80) Filter +(81) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(81) BroadcastExchange +(82) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(83) Project +(84) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(84) Scan parquet +(85) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(86) BroadcastExchange +(87) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(88) Project +(89) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(89) Scan parquet +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(91) BroadcastExchange +(92) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(92) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(93) Project +(94) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(94) HashAggregate +(95) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(95) Exchange +(96) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) HashAggregate +(97) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(97) Exchange +(98) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Sort +(99) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(99) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/1.txt index 1c8670c0fea4..63b7d317f3cf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (29) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (28) - +- Exchange (27) - +- HashAggregate (26) - +- Exchange (25) - +- HashAggregate (24) - +- Project (23) - +- Filter (22) - +- Scan parquet (21) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -57,101 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) VeloxAppendBatches +(14) VeloxAppendBatches Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) ColumnarExchange +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(18) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(20) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(21) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(22) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(23) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(24) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(25) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(27) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(29) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt index 61a6de21a400..db7c1cb79667 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt @@ -10,7 +10,7 @@ AdaptiveSparkPlan (68) +- ColumnarExchange (36) +- VeloxAppendBatches (35) +- ^ ProjectExecTransformer (33) - +- ^ RegularHashAggregateExecTransformer (32) + +- ^ FlushableHashAggregateExecTransformer (32) +- ^ ProjectExecTransformer (31) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) :- ^ ProjectExecTransformer (22) @@ -195,7 +195,7 @@ Join condition: None Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(32) RegularHashAggregateExecTransformer +(32) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt index 72b0287a9dbe..33c7971d3749 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt @@ -1,56 +1,57 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- ^ SortExecTransformer (37) - +- ^ InputIteratorTransformer (36) - +- ShuffleQueryStage (34) - +- ColumnarExchange (33) - +- VeloxAppendBatches (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7) - : +- ColumnarBroadcastExchange (6) - : +- ^ FilterExecTransformer (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FilterExecTransformer (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ FilterExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7) + : +- ColumnarBroadcastExchange (6) + : +- ^ FilterExecTransformer (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FilterExecTransformer (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (58) - +- Exchange (57) - +- Filter (56) - +- HashAggregate (55) - +- Exchange (54) - +- HashAggregate (53) - +- Project (52) - +- BroadcastHashJoin Inner BuildRight (51) - :- Project (46) - : +- BroadcastHashJoin Inner BuildRight (45) - : :- Filter (41) - : : +- Scan parquet (40) - : +- BroadcastExchange (44) - : +- Filter (43) - : +- Scan parquet (42) - +- BroadcastExchange (50) - +- Project (49) - +- Filter (48) - +- Scan parquet (47) + Sort (59) + +- Exchange (58) + +- Filter (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- BroadcastHashJoin Inner BuildRight (52) + :- Project (47) + : +- BroadcastHashJoin Inner BuildRight (46) + : :- Filter (42) + : : +- Scan parquet (41) + : +- BroadcastExchange (45) + : +- Filter (44) + : +- Scan parquet (43) + +- BroadcastExchange (51) + +- Project (50) + +- Filter (49) + +- Scan parquet (48) (1) Scan parquet @@ -159,157 +160,161 @@ Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(25) ColumnarExchange +(25) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(26) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(27) InputAdapter +(28) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(28) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(29) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(30) FilterExecTransformer +(31) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) VeloxAppendBatches +(33) VeloxAppendBatches Input [2]: [ps_partkey#X, value#X] Arguments: X -(33) ColumnarExchange +(34) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(34) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(35) InputAdapter +(36) InputAdapter Input [2]: [ps_partkey#X, value#X] -(36) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(37) SortExecTransformer +(38) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(38) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(39) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(40) Scan parquet +(41) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(41) Filter +(42) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(42) Scan parquet +(43) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(43) Filter +(44) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(44) BroadcastExchange +(45) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(45) BroadcastHashJoin +(46) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(46) Project +(47) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(47) Scan parquet +(48) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(48) Filter +(49) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(49) Project +(50) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(50) BroadcastExchange +(51) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(51) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(52) Project +(53) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(53) HashAggregate +(54) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(54) Exchange +(55) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) HashAggregate +(56) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(56) Filter +(57) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(57) Exchange +(58) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) Sort +(59) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(59) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt index 23ab061662fc..f1f3a9234354 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt @@ -1,42 +1,43 @@ == Physical Plan == -AdaptiveSparkPlan (43) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- VeloxAppendBatches (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (42) - +- Exchange (41) - +- HashAggregate (40) - +- Exchange (39) - +- HashAggregate (38) - +- Project (37) - +- BroadcastHashJoin Inner BuildLeft (36) - :- BroadcastExchange (32) - : +- Filter (31) - : +- Scan parquet (30) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (43) + +- Exchange (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- BroadcastHashJoin Inner BuildLeft (37) + :- BroadcastExchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -107,125 +108,129 @@ Input [3]: [l_shipmode#X, sum#X, sum#X] Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(16) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(20) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) VeloxAppendBatches +(23) VeloxAppendBatches Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(23) ColumnarExchange +(24) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(27) SortExecTransformer +(28) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(30) Scan parquet +(31) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(32) BroadcastExchange +(33) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(33) Scan parquet +(34) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(34) Filter +(35) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(35) Project +(36) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(36) BroadcastHashJoin +(37) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(37) Project +(38) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(38) HashAggregate +(39) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(39) Exchange +(40) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) HashAggregate +(41) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(41) Exchange +(42) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Sort +(43) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(43) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt index 0f55c5334af5..13ef25b681c0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt @@ -1,50 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32) - +- ColumnarExchange (31) - +- VeloxAppendBatches (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FilterExecTransformer (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ FilterExecTransformer (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (52) - +- Exchange (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- BroadcastHashJoin LeftOuter BuildRight (43) - :- Scan parquet (38) - +- BroadcastExchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- BroadcastHashJoin LeftOuter BuildRight (45) + :- Scan parquet (40) + +- BroadcastExchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -110,178 +112,186 @@ Input [2]: [c_custkey#X, count#X] Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, count#X] +Arguments: X + +(16) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [c_custkey#X, count#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [c_custkey#X, count#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(22) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) VeloxAppendBatches +(32) VeloxAppendBatches Input [2]: [c_count#X, custdist#X] Arguments: X -(31) ColumnarExchange +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(33) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(34) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(35) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(36) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(37) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(38) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(39) Scan parquet +(41) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(42) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(43) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) BroadcastExchange +(44) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(43) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(46) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) Exchange +(48) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(48) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(49) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(51) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(53) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt index 2427b05691d0..c86f2215cda5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt @@ -8,7 +8,7 @@ AdaptiveSparkPlan (35) +- ShuffleQueryStage (17) +- ColumnarExchange (16) +- VeloxAppendBatches (15) - +- ^ RegularHashAggregateExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) @@ -86,7 +86,7 @@ Join condition: None Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(13) RegularHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt index 64f59fa9c811..8edb179e592f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt @@ -1,44 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (46) +- == Final Plan == - VeloxColumnarToRowExec (30) - +- ^ SortExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- VeloxAppendBatches (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) + VeloxColumnarToRowExec (31) + +- ^ SortExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (21) :- ^ InputIteratorTransformer (7) : +- BroadcastQueryStage (5) : +- ColumnarBroadcastExchange (4) : +- ^ FilterExecTransformer (2) : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- Project (42) - +- BroadcastHashJoin Inner BuildLeft (41) - :- BroadcastExchange (33) - : +- Filter (32) - : +- Scan parquet (31) - +- Filter (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- Filter (35) - +- Scan parquet (34) + Sort (45) + +- Exchange (44) + +- Project (43) + +- BroadcastHashJoin Inner BuildLeft (42) + :- BroadcastExchange (34) + : +- Filter (33) + : +- Scan parquet (32) + +- Filter (41) + +- HashAggregate (40) + +- Exchange (39) + +- HashAggregate (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) (1) Scan parquet @@ -100,142 +101,146 @@ Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) VeloxAppendBatches +(24) VeloxAppendBatches Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(24) ColumnarExchange +(25) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) SortExecTransformer +(29) SortExecTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(29) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(30) VeloxColumnarToRowExec +(31) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(31) Scan parquet +(32) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(32) Filter +(33) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(33) BroadcastExchange +(34) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(34) Scan parquet +(35) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(35) Filter +(36) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(36) Project +(37) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(37) HashAggregate +(38) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(38) Exchange +(39) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(40) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(40) Filter +(41) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(42) Project +(43) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(43) Exchange +(44) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(45) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(46) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt index 7411f2a6667a..4d4d52c7d2e6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt @@ -1,54 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- VeloxAppendBatches (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ FilterExecTransformer (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ FilterExecTransformer (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (56) - +- Exchange (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- BroadcastHashJoin LeftAnti BuildRight (43) - : :- Filter (38) - : : +- Scan parquet (37) - : +- BroadcastExchange (42) - : +- Project (41) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Filter (45) - +- Scan parquet (44) + Sort (58) + +- Exchange (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Project (50) + +- BroadcastHashJoin Inner BuildRight (49) + :- BroadcastHashJoin LeftAnti BuildRight (45) + : :- Filter (40) + : : +- Scan parquet (39) + : +- BroadcastExchange (44) + : +- Project (43) + : +- Filter (42) + : +- Scan parquet (41) + +- BroadcastExchange (48) + +- Filter (47) + +- Scan parquet (46) (1) Scan parquet @@ -115,199 +117,207 @@ Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(16) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(25) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(26) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) VeloxAppendBatches +(31) VeloxAppendBatches Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(30) ColumnarExchange +(32) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(32) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(34) SortExecTransformer +(36) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(37) Scan parquet +(39) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(38) Filter +(40) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(39) Scan parquet +(41) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(40) Filter +(42) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(41) Project +(43) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(42) BroadcastExchange +(44) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(43) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(44) Scan parquet +(46) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(45) Filter +(47) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(46) BroadcastExchange +(48) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(47) BroadcastHashJoin +(49) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(48) Project +(50) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(49) HashAggregate +(51) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(50) Exchange +(52) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(52) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(53) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(56) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(55) Exchange +(57) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) Sort +(58) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(57) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/17.txt index 1b258da06f20..7c5359849d4e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/17.txt @@ -8,7 +8,7 @@ AdaptiveSparkPlan (36) +- ShuffleQueryStage (9) +- ColumnarExchange (8) +- VeloxAppendBatches (7) - +- ^ RegularHashAggregateExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (5) +- ^ InputIteratorTransformer (4) +- RowToVeloxColumnar (2) +- LocalTableScan (1) @@ -48,7 +48,7 @@ Input [1]: [l_extendedprice#X] (4) InputIteratorTransformer Input [1]: [l_extendedprice#X] -(5) RegularHashAggregateExecTransformer +(5) FlushableHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt index ced5a3b68cc4..a4aaf08ff8bb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt @@ -9,7 +9,7 @@ AdaptiveSparkPlan (88) +- ColumnarExchange (48) +- VeloxAppendBatches (47) +- ^ ProjectExecTransformer (45) - +- ^ RegularHashAggregateExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (44) +- ^ ProjectExecTransformer (43) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) :- ^ ProjectExecTransformer (29) @@ -33,7 +33,7 @@ AdaptiveSparkPlan (88) : +- ColumnarExchange (15) : +- VeloxAppendBatches (14) : +- ^ ProjectExecTransformer (12) - : +- ^ RegularHashAggregateExecTransformer (11) + : +- ^ FlushableHashAggregateExecTransformer (11) : +- ^ Scan parquet (10) +- ^ InputIteratorTransformer (41) +- BroadcastQueryStage (39) @@ -125,7 +125,7 @@ Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) RegularHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] @@ -261,7 +261,7 @@ Join condition: None Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(44) RegularHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt index 02f21577ad17..87acf3c4b28e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt @@ -7,7 +7,7 @@ AdaptiveSparkPlan (34) +- ShuffleQueryStage (17) +- ColumnarExchange (16) +- VeloxAppendBatches (15) - +- ^ RegularHashAggregateExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) @@ -85,7 +85,7 @@ Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) RegularHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt index c5b3f4cc9283..7b494469aacc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt @@ -1,16 +1,16 @@ == Physical Plan == -AdaptiveSparkPlan (109) +AdaptiveSparkPlan (110) +- == Final Plan == - VeloxColumnarToRowExec (72) - +- ^ SortExecTransformer (70) - +- ^ InputIteratorTransformer (69) - +- ShuffleQueryStage (67) - +- ColumnarExchange (66) - +- VeloxAppendBatches (65) - +- ^ ProjectExecTransformer (63) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (62) - :- ^ ProjectExecTransformer (53) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (52) + VeloxColumnarToRowExec (73) + +- ^ SortExecTransformer (71) + +- ^ InputIteratorTransformer (70) + +- ShuffleQueryStage (68) + +- ColumnarExchange (67) + +- VeloxAppendBatches (66) + +- ^ ProjectExecTransformer (64) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (63) + :- ^ ProjectExecTransformer (54) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (53) : :- ^ InputIteratorTransformer (10) : : +- AQEShuffleRead (8) : : +- ShuffleQueryStage (7) @@ -19,11 +19,11 @@ AdaptiveSparkPlan (109) : : +- ^ ProjectExecTransformer (3) : : +- ^ FilterExecTransformer (2) : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (51) - : +- BroadcastQueryStage (49) - : +- ColumnarBroadcastExchange (48) - : +- ^ ProjectExecTransformer (46) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (45) + : +- ^ InputIteratorTransformer (52) + : +- BroadcastQueryStage (50) + : +- ColumnarBroadcastExchange (49) + : +- ^ ProjectExecTransformer (47) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (46) : :- ^ InputIteratorTransformer (26) : : +- BroadcastQueryStage (24) : : +- ColumnarBroadcastExchange (23) @@ -36,64 +36,65 @@ AdaptiveSparkPlan (109) : : +- ^ ProjectExecTransformer (15) : : +- ^ FilterExecTransformer (14) : : +- ^ Scan parquet (13) - : +- ^ FilterExecTransformer (44) - : +- ^ ProjectExecTransformer (43) - : +- ^ RegularHashAggregateExecTransformer (42) - : +- ^ InputIteratorTransformer (41) - : +- ShuffleQueryStage (39) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ FlushableHashAggregateExecTransformer (35) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (34) - : :- ^ ProjectExecTransformer (29) - : : +- ^ FilterExecTransformer (28) - : : +- ^ Scan parquet (27) - : +- ^ InputIteratorTransformer (33) - : +- BroadcastQueryStage (31) - : +- ReusedExchange (30) - +- ^ InputIteratorTransformer (61) - +- BroadcastQueryStage (59) - +- ColumnarBroadcastExchange (58) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ Scan parquet (54) + : +- ^ FilterExecTransformer (45) + : +- ^ ProjectExecTransformer (44) + : +- ^ RegularHashAggregateExecTransformer (43) + : +- ^ InputIteratorTransformer (42) + : +- ShuffleQueryStage (40) + : +- ColumnarExchange (39) + : +- VeloxAppendBatches (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ FlushableHashAggregateExecTransformer (35) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (34) + : :- ^ ProjectExecTransformer (29) + : : +- ^ FilterExecTransformer (28) + : : +- ^ Scan parquet (27) + : +- ^ InputIteratorTransformer (33) + : +- BroadcastQueryStage (31) + : +- ReusedExchange (30) + +- ^ InputIteratorTransformer (62) + +- BroadcastQueryStage (60) + +- ColumnarBroadcastExchange (59) + +- ^ ProjectExecTransformer (57) + +- ^ FilterExecTransformer (56) + +- ^ Scan parquet (55) +- == Initial Plan == - Sort (108) - +- Exchange (107) - +- Project (106) - +- BroadcastHashJoin Inner BuildRight (105) - :- Project (100) - : +- ShuffledHashJoin LeftSemi BuildRight (99) - : :- Exchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- Exchange (98) - : +- Project (97) - : +- BroadcastHashJoin Inner BuildLeft (96) - : :- BroadcastExchange (83) - : : +- BroadcastHashJoin LeftSemi BuildRight (82) - : : :- Filter (77) - : : : +- Scan parquet (76) - : : +- BroadcastExchange (81) - : : +- Project (80) - : : +- Filter (79) - : : +- Scan parquet (78) - : +- Filter (95) - : +- HashAggregate (94) - : +- Exchange (93) - : +- HashAggregate (92) - : +- BroadcastHashJoin LeftSemi BuildRight (91) - : :- Project (86) - : : +- Filter (85) - : : +- Scan parquet (84) - : +- BroadcastExchange (90) - : +- Project (89) - : +- Filter (88) - : +- Scan parquet (87) - +- BroadcastExchange (104) - +- Project (103) - +- Filter (102) - +- Scan parquet (101) + Sort (109) + +- Exchange (108) + +- Project (107) + +- BroadcastHashJoin Inner BuildRight (106) + :- Project (101) + : +- ShuffledHashJoin LeftSemi BuildRight (100) + : :- Exchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Exchange (99) + : +- Project (98) + : +- BroadcastHashJoin Inner BuildLeft (97) + : :- BroadcastExchange (84) + : : +- BroadcastHashJoin LeftSemi BuildRight (83) + : : :- Filter (78) + : : : +- Scan parquet (77) + : : +- BroadcastExchange (82) + : : +- Project (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- Filter (96) + : +- HashAggregate (95) + : +- Exchange (94) + : +- HashAggregate (93) + : +- BroadcastHashJoin LeftSemi BuildRight (92) + : :- Project (87) + : : +- Filter (86) + : : +- Scan parquet (85) + : +- BroadcastExchange (91) + : +- Project (90) + : +- Filter (89) + : +- Scan parquet (88) + +- BroadcastExchange (105) + +- Project (104) + +- Filter (103) + +- Scan parquet (102) (1) Scan parquet @@ -252,319 +253,323 @@ Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(38) ColumnarExchange +(38) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(39) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(40) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(40) InputAdapter +(41) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(41) InputIteratorTransformer +(42) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(42) RegularHashAggregateExecTransformer +(43) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) ProjectExecTransformer +(44) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(44) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(45) BroadcastHashJoinExecTransformer +(46) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(46) ProjectExecTransformer +(47) ProjectExecTransformer Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(47) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] Arguments: false -(48) ColumnarBroadcastExchange +(49) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(49) BroadcastQueryStage +(50) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [1]: [ps_suppkey#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(52) BroadcastHashJoinExecTransformer +(53) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(53) ProjectExecTransformer +(54) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(54) Scan parquet +(55) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(55) FilterExecTransformer +(56) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(56) ProjectExecTransformer +(57) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(58) ColumnarBroadcastExchange +(59) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(59) BroadcastQueryStage +(60) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(60) InputAdapter +(61) InputAdapter Input [1]: [n_nationkey#X] -(61) InputIteratorTransformer +(62) InputIteratorTransformer Input [1]: [n_nationkey#X] -(62) BroadcastHashJoinExecTransformer +(63) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(64) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(65) VeloxAppendBatches +(66) VeloxAppendBatches Input [2]: [s_name#X, s_address#X] Arguments: X -(66) ColumnarExchange +(67) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(68) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(68) InputAdapter +(69) InputAdapter Input [2]: [s_name#X, s_address#X] -(69) InputIteratorTransformer +(70) InputIteratorTransformer Input [2]: [s_name#X, s_address#X] -(70) SortExecTransformer +(71) SortExecTransformer Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(71) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(72) VeloxColumnarToRowExec +(73) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(73) Scan parquet +(74) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(75) Exchange +(76) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) Scan parquet +(77) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(77) Filter +(78) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(78) Scan parquet +(79) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(79) Filter +(80) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(80) Project +(81) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(81) BroadcastExchange +(82) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(83) BroadcastExchange +(84) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(84) Scan parquet +(85) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(86) Project +(87) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(87) Scan parquet +(88) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(88) Filter +(89) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(89) Project +(90) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(90) BroadcastExchange +(91) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(92) HashAggregate +(93) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(93) Exchange +(94) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) HashAggregate +(95) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(95) Filter +(96) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(96) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(97) Project +(98) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(98) Exchange +(99) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) ShuffledHashJoin +(100) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(100) Project +(101) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(101) Scan parquet +(102) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(102) Filter +(103) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(103) Project +(104) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(104) BroadcastExchange +(105) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(105) BroadcastHashJoin +(106) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(106) Project +(107) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(107) Exchange +(108) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Sort +(109) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(109) AdaptiveSparkPlan +(110) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt index d22e64b92b4b..ee28d7b592e8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt @@ -9,7 +9,7 @@ AdaptiveSparkPlan (93) +- ColumnarExchange (53) +- VeloxAppendBatches (52) +- ^ ProjectExecTransformer (50) - +- ^ RegularHashAggregateExecTransformer (49) + +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) :- ^ ProjectExecTransformer (38) @@ -287,7 +287,7 @@ Join condition: None Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(49) RegularHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt index 9344c1bf2f70..d578b43f3d6a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt @@ -1,38 +1,39 @@ == Physical Plan == -AdaptiveSparkPlan (39) +AdaptiveSparkPlan (40) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- ^ SortExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22) - +- ColumnarExchange (21) - +- VeloxAppendBatches (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (28) + +- ^ SortExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- VeloxAppendBatches (21) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (38) - +- Exchange (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- BroadcastHashJoin LeftAnti BuildRight (32) - :- Filter (29) - : +- Scan parquet (28) - +- BroadcastExchange (31) - +- Scan parquet (30) + Sort (39) + +- Exchange (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- BroadcastHashJoin LeftAnti BuildRight (33) + :- Filter (30) + : +- Scan parquet (29) + +- BroadcastExchange (32) + +- Scan parquet (31) (1) Scan parquet @@ -94,116 +95,120 @@ Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) VeloxAppendBatches +(21) VeloxAppendBatches Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(21) ColumnarExchange +(22) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(23) InputAdapter +(24) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(25) SortExecTransformer +(26) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(26) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(28) Scan parquet +(29) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(29) Filter +(30) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(30) Scan parquet +(31) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(31) BroadcastExchange +(32) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(32) BroadcastHashJoin +(33) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(33) Project +(34) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(34) HashAggregate +(35) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(35) Exchange +(36) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(37) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(37) Exchange +(38) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Sort +(39) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(39) AdaptiveSparkPlan +(40) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt index 3cd44f2a745a..2ffd75b92964 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt @@ -10,7 +10,7 @@ AdaptiveSparkPlan (54) +- ColumnarExchange (27) +- VeloxAppendBatches (26) +- ^ ProjectExecTransformer (24) - +- ^ RegularHashAggregateExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (23) +- ^ ProjectExecTransformer (22) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) :- ^ ProjectExecTransformer (12) @@ -145,7 +145,7 @@ Join condition: None Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) RegularHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt index 703f0f47e882..36c6de5374ba 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt @@ -1,44 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (46) +- == Final Plan == - VeloxColumnarToRowExec (30) - +- ^ SortExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- VeloxAppendBatches (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (31) + +- ^ SortExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ InputIteratorTransformer (21) + +- ShuffleQueryStage (19) + +- ColumnarExchange (18) + +- VeloxAppendBatches (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- HashAggregate (42) - +- Exchange (41) - +- HashAggregate (40) - +- Project (39) - +- BroadcastHashJoin LeftSemi BuildRight (38) - :- Project (33) - : +- Filter (32) - : +- Scan parquet (31) - +- BroadcastExchange (37) - +- Project (36) - +- Filter (35) - +- Scan parquet (34) + Sort (45) + +- Exchange (44) + +- HashAggregate (43) + +- Exchange (42) + +- HashAggregate (41) + +- Project (40) + +- BroadcastHashJoin LeftSemi BuildRight (39) + :- Project (34) + : +- Filter (33) + : +- Scan parquet (32) + +- BroadcastExchange (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) (1) Scan parquet @@ -113,129 +114,133 @@ Input [2]: [o_orderpriority#X, count#X] Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(17) ColumnarExchange +(17) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(18) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(18) ShuffleQueryStage +(19) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(19) InputAdapter +(20) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(20) InputIteratorTransformer +(21) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(21) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) VeloxAppendBatches +(24) VeloxAppendBatches Input [2]: [o_orderpriority#X, order_count#X] Arguments: X -(24) ColumnarExchange +(25) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(28) SortExecTransformer +(29) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(29) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(30) VeloxColumnarToRowExec +(31) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(31) Scan parquet +(32) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(32) Filter +(33) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(33) Project +(34) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(34) Scan parquet +(35) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(35) Filter +(36) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(36) Project +(37) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(37) BroadcastExchange +(38) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(38) BroadcastHashJoin +(39) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(39) Project +(40) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(40) HashAggregate +(41) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(41) Exchange +(42) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) HashAggregate +(43) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(43) Exchange +(44) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(45) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(46) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt index 56a69c4dad35..49ee27485b4a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt @@ -1,92 +1,93 @@ == Physical Plan == -AdaptiveSparkPlan (101) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (66) - +- ^ SortExecTransformer (64) - +- ^ InputIteratorTransformer (63) - +- ShuffleQueryStage (61) - +- ColumnarExchange (60) - +- VeloxAppendBatches (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (67) + +- ^ SortExecTransformer (65) + +- ^ InputIteratorTransformer (64) + +- ShuffleQueryStage (62) + +- ColumnarExchange (61) + +- VeloxAppendBatches (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ InputIteratorTransformer (57) + +- ShuffleQueryStage (55) + +- ColumnarExchange (54) + +- VeloxAppendBatches (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (100) - +- Exchange (99) - +- HashAggregate (98) - +- Exchange (97) - +- HashAggregate (96) - +- Project (95) - +- BroadcastHashJoin Inner BuildRight (94) - :- Project (89) - : +- BroadcastHashJoin Inner BuildRight (88) - : :- Project (84) - : : +- BroadcastHashJoin Inner BuildRight (83) - : : :- Project (79) - : : : +- BroadcastHashJoin Inner BuildRight (78) - : : : :- Project (74) - : : : : +- BroadcastHashJoin Inner BuildLeft (73) - : : : : :- BroadcastExchange (69) - : : : : : +- Filter (68) - : : : : : +- Scan parquet (67) - : : : : +- Project (72) - : : : : +- Filter (71) - : : : : +- Scan parquet (70) - : : : +- BroadcastExchange (77) - : : : +- Filter (76) - : : : +- Scan parquet (75) - : : +- BroadcastExchange (82) - : : +- Filter (81) - : : +- Scan parquet (80) - : +- BroadcastExchange (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (93) - +- Project (92) - +- Filter (91) - +- Scan parquet (90) + Sort (101) + +- Exchange (100) + +- HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Project (96) + +- BroadcastHashJoin Inner BuildRight (95) + :- Project (90) + : +- BroadcastHashJoin Inner BuildRight (89) + : :- Project (85) + : : +- BroadcastHashJoin Inner BuildRight (84) + : : :- Project (80) + : : : +- BroadcastHashJoin Inner BuildRight (79) + : : : :- Project (75) + : : : : +- BroadcastHashJoin Inner BuildLeft (74) + : : : : :- BroadcastExchange (70) + : : : : : +- Filter (69) + : : : : : +- Scan parquet (68) + : : : : +- Project (73) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (78) + : : : +- Filter (77) + : : : +- Scan parquet (76) + : : +- BroadcastExchange (83) + : : +- Filter (82) + : : +- Scan parquet (81) + : +- BroadcastExchange (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (94) + +- Project (93) + +- Filter (92) + +- Scan parquet (91) (1) Scan parquet @@ -313,225 +314,229 @@ Input [3]: [n_name#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(53) ColumnarExchange +(53) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(54) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(55) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(55) InputAdapter +(56) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(56) InputIteratorTransformer +(57) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(57) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(58) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(59) VeloxAppendBatches +(60) VeloxAppendBatches Input [2]: [n_name#X, revenue#X] Arguments: X -(60) ColumnarExchange +(61) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(62) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(62) InputAdapter +(63) InputAdapter Input [2]: [n_name#X, revenue#X] -(63) InputIteratorTransformer +(64) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(64) SortExecTransformer +(65) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(65) WholeStageCodegenTransformer (X) +(66) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(66) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(67) Scan parquet +(68) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(68) Filter +(69) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(69) BroadcastExchange +(70) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(70) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(72) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(74) Project +(75) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(75) Scan parquet +(76) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(76) Filter +(77) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(77) BroadcastExchange +(78) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(78) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(79) Project +(80) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(80) Scan parquet +(81) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(81) Filter +(82) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(82) BroadcastExchange +(83) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(83) BroadcastHashJoin +(84) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(84) Project +(85) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(85) Scan parquet +(86) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(87) BroadcastExchange +(88) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(89) Project +(90) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(90) Scan parquet +(91) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(91) Filter +(92) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(92) Project +(93) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(93) BroadcastExchange +(94) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(94) BroadcastHashJoin +(95) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(95) Project +(96) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(96) HashAggregate +(97) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(97) Exchange +(98) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) HashAggregate +(99) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(99) Exchange +(100) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Sort +(101) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(101) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/6.txt index 41f69e74d1a0..786a89fe715a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/6.txt @@ -7,7 +7,7 @@ AdaptiveSparkPlan (20) +- ShuffleQueryStage (8) +- ColumnarExchange (7) +- VeloxAppendBatches (6) - +- ^ RegularHashAggregateExecTransformer (4) + +- ^ FlushableHashAggregateExecTransformer (4) +- ^ ProjectExecTransformer (3) +- ^ FilterExecTransformer (2) +- ^ Scan parquet (1) @@ -35,7 +35,7 @@ Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnot Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) RegularHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt index 5e99678cabb7..2ba42f806f3e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt @@ -1,86 +1,87 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (95) +- == Final Plan == - VeloxColumnarToRowExec (61) - +- ^ SortExecTransformer (59) - +- ^ InputIteratorTransformer (58) - +- ShuffleQueryStage (56) - +- ColumnarExchange (55) - +- VeloxAppendBatches (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ FilterExecTransformer (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ FilterExecTransformer (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34) - : +- ColumnarBroadcastExchange (33) - : +- ^ FilterExecTransformer (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40) - +- ReusedExchange (39) + VeloxColumnarToRowExec (62) + +- ^ SortExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57) + +- ColumnarExchange (56) + +- VeloxAppendBatches (55) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ FilterExecTransformer (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ FilterExecTransformer (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40) + +- ReusedExchange (39) +- == Initial Plan == - Sort (93) - +- Exchange (92) - +- HashAggregate (91) - +- Exchange (90) - +- HashAggregate (89) - +- Project (88) - +- BroadcastHashJoin Inner BuildRight (87) - :- Project (83) - : +- BroadcastHashJoin Inner BuildRight (82) - : :- Project (78) - : : +- BroadcastHashJoin Inner BuildRight (77) - : : :- Project (73) - : : : +- BroadcastHashJoin Inner BuildRight (72) - : : : :- Project (68) - : : : : +- BroadcastHashJoin Inner BuildLeft (67) - : : : : :- BroadcastExchange (64) - : : : : : +- Filter (63) - : : : : : +- Scan parquet (62) - : : : : +- Filter (66) - : : : : +- Scan parquet (65) - : : : +- BroadcastExchange (71) - : : : +- Filter (70) - : : : +- Scan parquet (69) - : : +- BroadcastExchange (76) - : : +- Filter (75) - : : +- Scan parquet (74) - : +- BroadcastExchange (81) - : +- Filter (80) - : +- Scan parquet (79) - +- BroadcastExchange (86) - +- Filter (85) - +- Scan parquet (84) + Sort (94) + +- Exchange (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- BroadcastHashJoin Inner BuildRight (88) + :- Project (84) + : +- BroadcastHashJoin Inner BuildRight (83) + : :- Project (79) + : : +- BroadcastHashJoin Inner BuildRight (78) + : : :- Project (74) + : : : +- BroadcastHashJoin Inner BuildRight (73) + : : : :- Project (69) + : : : : +- BroadcastHashJoin Inner BuildLeft (68) + : : : : :- BroadcastExchange (65) + : : : : : +- Filter (64) + : : : : : +- Scan parquet (63) + : : : : +- Filter (67) + : : : : +- Scan parquet (66) + : : : +- BroadcastExchange (72) + : : : +- Filter (71) + : : : +- Scan parquet (70) + : : +- BroadcastExchange (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- BroadcastExchange (82) + : +- Filter (81) + : +- Scan parquet (80) + +- BroadcastExchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -283,217 +284,221 @@ Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(48) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(49) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(50) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(53) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) VeloxAppendBatches +(55) VeloxAppendBatches Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(55) ColumnarExchange +(56) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(56) ShuffleQueryStage +(57) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(57) InputAdapter +(58) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) InputIteratorTransformer +(59) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(59) SortExecTransformer +(60) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(60) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(61) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(62) Scan parquet +(63) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(63) Filter +(64) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(64) BroadcastExchange +(65) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(65) Scan parquet +(66) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(67) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(68) Project +(69) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(69) Scan parquet +(70) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(71) BroadcastExchange +(72) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(73) Project +(74) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(75) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(78) Project +(79) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(79) Scan parquet +(80) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(80) Filter +(81) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(81) BroadcastExchange +(82) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(83) Project +(84) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(84) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(85) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(86) BroadcastExchange +(87) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(88) Project +(89) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(89) HashAggregate +(90) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(90) Exchange +(91) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(92) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(92) Exchange +(93) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Sort +(94) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(94) AdaptiveSparkPlan +(95) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt index f008d04b81aa..64a40563c238 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt @@ -1,117 +1,118 @@ == Physical Plan == -AdaptiveSparkPlan (130) +AdaptiveSparkPlan (131) +- == Final Plan == - VeloxColumnarToRowExec (85) - +- ^ SortExecTransformer (83) - +- ^ InputIteratorTransformer (82) - +- ShuffleQueryStage (80) - +- ColumnarExchange (79) - +- VeloxAppendBatches (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ FilterExecTransformer (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ FilterExecTransformer (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ FilterExecTransformer (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ FilterExecTransformer (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53) - : +- ColumnarBroadcastExchange (52) - : +- ^ FilterExecTransformer (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ FilterExecTransformer (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (86) + +- ^ SortExecTransformer (84) + +- ^ InputIteratorTransformer (83) + +- ShuffleQueryStage (81) + +- ColumnarExchange (80) + +- VeloxAppendBatches (79) + +- ^ ProjectExecTransformer (77) + +- ^ RegularHashAggregateExecTransformer (76) + +- ^ InputIteratorTransformer (75) + +- ShuffleQueryStage (73) + +- ColumnarExchange (72) + +- VeloxAppendBatches (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ FilterExecTransformer (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ FilterExecTransformer (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ FilterExecTransformer (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ FilterExecTransformer (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ FilterExecTransformer (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53) + : +- ColumnarBroadcastExchange (52) + : +- ^ FilterExecTransformer (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ FilterExecTransformer (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (129) - +- Exchange (128) - +- HashAggregate (127) - +- Exchange (126) - +- HashAggregate (125) - +- Project (124) - +- BroadcastHashJoin Inner BuildRight (123) - :- Project (118) - : +- BroadcastHashJoin Inner BuildRight (117) - : :- Project (113) - : : +- BroadcastHashJoin Inner BuildRight (112) - : : :- Project (108) - : : : +- BroadcastHashJoin Inner BuildRight (107) - : : : :- Project (103) - : : : : +- BroadcastHashJoin Inner BuildRight (102) - : : : : :- Project (98) - : : : : : +- BroadcastHashJoin Inner BuildRight (97) - : : : : : :- Project (93) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (92) - : : : : : : :- BroadcastExchange (89) - : : : : : : : +- Project (88) - : : : : : : : +- Filter (87) - : : : : : : : +- Scan parquet (86) - : : : : : : +- Filter (91) - : : : : : : +- Scan parquet (90) - : : : : : +- BroadcastExchange (96) - : : : : : +- Filter (95) - : : : : : +- Scan parquet (94) - : : : : +- BroadcastExchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- BroadcastExchange (106) - : : : +- Filter (105) - : : : +- Scan parquet (104) - : : +- BroadcastExchange (111) - : : +- Filter (110) - : : +- Scan parquet (109) - : +- BroadcastExchange (116) - : +- Filter (115) - : +- Scan parquet (114) - +- BroadcastExchange (122) - +- Project (121) - +- Filter (120) - +- Scan parquet (119) + Sort (130) + +- Exchange (129) + +- HashAggregate (128) + +- Exchange (127) + +- HashAggregate (126) + +- Project (125) + +- BroadcastHashJoin Inner BuildRight (124) + :- Project (119) + : +- BroadcastHashJoin Inner BuildRight (118) + : :- Project (114) + : : +- BroadcastHashJoin Inner BuildRight (113) + : : :- Project (109) + : : : +- BroadcastHashJoin Inner BuildRight (108) + : : : :- Project (104) + : : : : +- BroadcastHashJoin Inner BuildRight (103) + : : : : :- Project (99) + : : : : : +- BroadcastHashJoin Inner BuildRight (98) + : : : : : :- Project (94) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (93) + : : : : : : :- BroadcastExchange (90) + : : : : : : : +- Project (89) + : : : : : : : +- Filter (88) + : : : : : : : +- Scan parquet (87) + : : : : : : +- Filter (92) + : : : : : : +- Scan parquet (91) + : : : : : +- BroadcastExchange (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- BroadcastExchange (102) + : : : : +- Filter (101) + : : : : +- Scan parquet (100) + : : : +- BroadcastExchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- BroadcastExchange (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- BroadcastExchange (117) + : +- Filter (116) + : +- Scan parquet (115) + +- BroadcastExchange (123) + +- Project (122) + +- Filter (121) + +- Scan parquet (120) (1) Scan parquet @@ -414,277 +415,281 @@ Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(71) ColumnarExchange +(71) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(72) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(72) ShuffleQueryStage +(73) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(73) InputAdapter +(74) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(74) InputIteratorTransformer +(75) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(75) RegularHashAggregateExecTransformer +(76) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(76) ProjectExecTransformer +(77) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(77) WholeStageCodegenTransformer (X) +(78) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) VeloxAppendBatches +(79) VeloxAppendBatches Input [2]: [o_year#X, mkt_share#X] Arguments: X -(79) ColumnarExchange +(80) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(80) ShuffleQueryStage +(81) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(81) InputAdapter +(82) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(82) InputIteratorTransformer +(83) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(83) SortExecTransformer +(84) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(84) WholeStageCodegenTransformer (X) +(85) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(85) VeloxColumnarToRowExec +(86) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(86) Scan parquet +(87) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(87) Filter +(88) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(88) Project +(89) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(89) BroadcastExchange +(90) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(90) Scan parquet +(91) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(91) Filter +(92) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(92) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(93) Project +(94) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(94) Scan parquet +(95) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(95) Filter +(96) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(96) BroadcastExchange +(97) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(97) BroadcastHashJoin +(98) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(98) Project +(99) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(99) Scan parquet +(100) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(100) Filter +(101) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(101) BroadcastExchange +(102) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(102) BroadcastHashJoin +(103) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(103) Project +(104) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(104) Scan parquet +(105) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(105) Filter +(106) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(106) BroadcastExchange +(107) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(107) BroadcastHashJoin +(108) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(108) Project +(109) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(109) Scan parquet +(110) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(110) Filter +(111) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(111) BroadcastExchange +(112) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(112) BroadcastHashJoin +(113) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(113) Project +(114) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(114) Scan parquet +(115) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(115) Filter +(116) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(116) BroadcastExchange +(117) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(117) BroadcastHashJoin +(118) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(118) Project +(119) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(119) Scan parquet +(120) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(120) Filter +(121) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(121) Project +(122) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(122) BroadcastExchange +(123) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(123) BroadcastHashJoin +(124) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(124) Project +(125) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(125) HashAggregate +(126) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(126) Exchange +(127) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) HashAggregate +(128) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(128) Exchange +(129) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Sort +(130) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(130) AdaptiveSparkPlan +(131) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt index dcfe2020f21c..b5a21a0261d2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt @@ -1,90 +1,91 @@ == Physical Plan == -AdaptiveSparkPlan (99) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60) - +- ColumnarExchange (59) - +- VeloxAppendBatches (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44) - +- ColumnarBroadcastExchange (43) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (98) - +- Exchange (97) - +- HashAggregate (96) - +- Exchange (95) - +- HashAggregate (94) - +- Project (93) - +- BroadcastHashJoin Inner BuildRight (92) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (69) - : : : : : +- Project (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Filter (71) - : : : : +- Scan parquet (70) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (91) - +- Filter (90) - +- Scan parquet (89) + Sort (99) + +- Exchange (98) + +- HashAggregate (97) + +- Exchange (96) + +- HashAggregate (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (70) + : : : : : +- Project (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -307,221 +308,225 @@ Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(53) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) VeloxAppendBatches +(59) VeloxAppendBatches Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(59) ColumnarExchange +(60) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(61) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(61) InputAdapter +(62) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) InputIteratorTransformer +(63) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(63) SortExecTransformer +(64) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(65) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(67) Filter +(68) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(68) Project +(69) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(69) BroadcastExchange +(70) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(70) Scan parquet +(71) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(73) Project +(74) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(74) Scan parquet +(75) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(78) Project +(79) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(79) Scan parquet +(80) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(80) Filter +(81) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(81) BroadcastExchange +(82) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(83) Project +(84) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(84) Scan parquet +(85) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(86) BroadcastExchange +(87) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(88) Project +(89) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(89) Scan parquet +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(91) BroadcastExchange +(92) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(92) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(93) Project +(94) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(94) HashAggregate +(95) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(95) Exchange +(96) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) HashAggregate +(97) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(97) Exchange +(98) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Sort +(99) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(99) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/1.txt index 2f3257bbdafc..22dd5100c4fb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (29) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (28) - +- Exchange (27) - +- HashAggregate (26) - +- Exchange (25) - +- HashAggregate (24) - +- Project (23) - +- Filter (22) - +- Scan parquet (21) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -57,101 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) VeloxAppendBatches +(14) VeloxAppendBatches Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) ColumnarExchange +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(18) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(20) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(21) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(22) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(23) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(24) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(25) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(27) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(29) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt index 7ccdb0e04514..8681ecf5f93f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt @@ -10,7 +10,7 @@ AdaptiveSparkPlan (68) +- ColumnarExchange (36) +- VeloxAppendBatches (35) +- ^ ProjectExecTransformer (33) - +- ^ RegularHashAggregateExecTransformer (32) + +- ^ FlushableHashAggregateExecTransformer (32) +- ^ ProjectExecTransformer (31) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) :- ^ ProjectExecTransformer (22) @@ -195,7 +195,7 @@ Join condition: None Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(32) RegularHashAggregateExecTransformer +(32) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt index 619379320e1c..6dfc1d00f4cc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt @@ -1,56 +1,57 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- ^ SortExecTransformer (37) - +- ^ InputIteratorTransformer (36) - +- ShuffleQueryStage (34), Statistics(X) - +- ColumnarExchange (33) - +- VeloxAppendBatches (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7), Statistics(X) - : +- ColumnarBroadcastExchange (6) - : +- ^ FilterExecTransformer (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17), Statistics(X) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FilterExecTransformer (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ FilterExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ FilterExecTransformer (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FilterExecTransformer (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (58) - +- Exchange (57) - +- Filter (56) - +- HashAggregate (55) - +- Exchange (54) - +- HashAggregate (53) - +- Project (52) - +- BroadcastHashJoin Inner BuildRight (51) - :- Project (46) - : +- BroadcastHashJoin Inner BuildRight (45) - : :- Filter (41) - : : +- Scan parquet (40) - : +- BroadcastExchange (44) - : +- Filter (43) - : +- Scan parquet (42) - +- BroadcastExchange (50) - +- Project (49) - +- Filter (48) - +- Scan parquet (47) + Sort (59) + +- Exchange (58) + +- Filter (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- BroadcastHashJoin Inner BuildRight (52) + :- Project (47) + : +- BroadcastHashJoin Inner BuildRight (46) + : :- Filter (42) + : : +- Scan parquet (41) + : +- BroadcastExchange (45) + : +- Filter (44) + : +- Scan parquet (43) + +- BroadcastExchange (51) + +- Project (50) + +- Filter (49) + +- Scan parquet (48) (1) Scan parquet @@ -159,383 +160,392 @@ Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(25) ColumnarExchange +(25) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(26) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(27) InputAdapter +(28) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(28) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(29) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(30) FilterExecTransformer +(31) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) VeloxAppendBatches +(33) VeloxAppendBatches Input [2]: [ps_partkey#X, value#X] Arguments: X -(33) ColumnarExchange +(34) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(34) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(35) InputAdapter +(36) InputAdapter Input [2]: [ps_partkey#X, value#X] -(36) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(37) SortExecTransformer +(38) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(38) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(39) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(40) Scan parquet +(41) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(41) Filter +(42) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(42) Scan parquet +(43) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(43) Filter +(44) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(44) BroadcastExchange +(45) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(45) BroadcastHashJoin +(46) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(46) Project +(47) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(47) Scan parquet +(48) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(48) Filter +(49) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(49) Project +(50) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(50) BroadcastExchange +(51) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(51) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(52) Project +(53) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(53) HashAggregate +(54) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(54) Exchange +(55) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) HashAggregate +(56) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(56) Filter +(57) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(57) Exchange +(58) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) Sort +(59) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(59) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 30 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (100) +Subquery:1 Hosting operator id = 31 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (83) - +- ^ ProjectExecTransformer (81) - +- ^ RegularHashAggregateExecTransformer (80) - +- ^ InputIteratorTransformer (79) - +- ShuffleQueryStage (77), Statistics(X) - +- ColumnarExchange (76) - +- ^ FlushableHashAggregateExecTransformer (74) - +- ^ ProjectExecTransformer (73) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (72) - :- ^ ProjectExecTransformer (67) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - : :- ^ FilterExecTransformer (61) - : : +- ^ Scan parquet (60) - : +- ^ InputIteratorTransformer (65) - : +- BroadcastQueryStage (63), Statistics(X) - : +- ReusedExchange (62) - +- ^ InputIteratorTransformer (71) - +- BroadcastQueryStage (69), Statistics(X) - +- ReusedExchange (68) + VeloxColumnarToRowExec (85) + +- ^ ProjectExecTransformer (83) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- VeloxAppendBatches (77) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (73) + :- ^ ProjectExecTransformer (68) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (67) + : :- ^ FilterExecTransformer (62) + : : +- ^ Scan parquet (61) + : +- ^ InputIteratorTransformer (66) + : +- BroadcastQueryStage (64), Statistics(X) + : +- ReusedExchange (63) + +- ^ InputIteratorTransformer (72) + +- BroadcastQueryStage (70), Statistics(X) + +- ReusedExchange (69) +- == Initial Plan == - HashAggregate (99) - +- Exchange (98) - +- HashAggregate (97) - +- Project (96) - +- BroadcastHashJoin Inner BuildRight (95) - :- Project (90) - : +- BroadcastHashJoin Inner BuildRight (89) - : :- Filter (85) - : : +- Scan parquet (84) - : +- BroadcastExchange (88) - : +- Filter (87) - : +- Scan parquet (86) - +- BroadcastExchange (94) - +- Project (93) - +- Filter (92) - +- Scan parquet (91) - - -(60) Scan parquet + HashAggregate (101) + +- Exchange (100) + +- HashAggregate (99) + +- Project (98) + +- BroadcastHashJoin Inner BuildRight (97) + :- Project (92) + : +- BroadcastHashJoin Inner BuildRight (91) + : :- Filter (87) + : : +- Scan parquet (86) + : +- BroadcastExchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- BroadcastExchange (96) + +- Project (95) + +- Filter (94) + +- Scan parquet (93) + + +(61) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(61) FilterExecTransformer +(62) FilterExecTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: isnotnull(ps_suppkey#X) -(62) ReusedExchange [Reuses operator id: 6] +(63) ReusedExchange [Reuses operator id: 6] Output [2]: [s_suppkey#X, s_nationkey#X] -(63) BroadcastQueryStage +(64) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(64) InputAdapter +(65) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(65) InputIteratorTransformer +(66) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(66) BroadcastHashJoinExecTransformer +(67) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(67) ProjectExecTransformer +(68) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(68) ReusedExchange [Reuses operator id: 16] +(69) ReusedExchange [Reuses operator id: 16] Output [1]: [n_nationkey#X] -(69) BroadcastQueryStage +(70) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(70) InputAdapter +(71) InputAdapter Input [1]: [n_nationkey#X] -(71) InputIteratorTransformer +(72) InputIteratorTransformer Input [1]: [n_nationkey#X] -(72) BroadcastHashJoinExecTransformer +(73) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(73) ProjectExecTransformer +(74) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(74) FlushableHashAggregateExecTransformer +(75) FlushableHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(75) WholeStageCodegenTransformer (X) +(76) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(76) ColumnarExchange +(77) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(78) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(77) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(78) InputAdapter +(80) InputAdapter Input [2]: [sum#X, isEmpty#X] -(79) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(80) RegularHashAggregateExecTransformer +(82) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(81) ProjectExecTransformer +(83) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(82) WholeStageCodegenTransformer (X) +(84) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(83) VeloxColumnarToRowExec +(85) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(84) Scan parquet +(86) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(85) Filter +(87) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(86) Scan parquet +(88) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(87) Filter +(89) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(88) BroadcastExchange +(90) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(89) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(90) Project +(92) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(91) Scan parquet +(93) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(92) Filter +(94) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(93) Project +(95) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(94) BroadcastExchange +(96) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(95) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(96) Project +(98) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(97) HashAggregate +(99) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(98) Exchange +(100) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(99) HashAggregate +(101) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(100) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt index efb7a9793b9e..1e1ad6d497a4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt @@ -1,42 +1,43 @@ == Physical Plan == -AdaptiveSparkPlan (43) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- VeloxAppendBatches (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (42) - +- Exchange (41) - +- HashAggregate (40) - +- Exchange (39) - +- HashAggregate (38) - +- Project (37) - +- BroadcastHashJoin Inner BuildLeft (36) - :- BroadcastExchange (32) - : +- Filter (31) - : +- Scan parquet (30) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (43) + +- Exchange (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- BroadcastHashJoin Inner BuildLeft (37) + :- BroadcastExchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -107,125 +108,129 @@ Input [3]: [l_shipmode#X, sum#X, sum#X] Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(16) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(20) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) VeloxAppendBatches +(23) VeloxAppendBatches Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(23) ColumnarExchange +(24) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(27) SortExecTransformer +(28) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(30) Scan parquet +(31) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(32) BroadcastExchange +(33) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(33) Scan parquet +(34) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(34) Filter +(35) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(35) Project +(36) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(36) BroadcastHashJoin +(37) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(37) Project +(38) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(38) HashAggregate +(39) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(39) Exchange +(40) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) HashAggregate +(41) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(41) Exchange +(42) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Sort +(43) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(43) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt index e75208be3e05..ff6532aa6579 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt @@ -1,50 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32), Statistics(X) - +- ColumnarExchange (31) - +- VeloxAppendBatches (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FilterExecTransformer (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ FilterExecTransformer (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (52) - +- Exchange (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- BroadcastHashJoin LeftOuter BuildRight (43) - :- Scan parquet (38) - +- BroadcastExchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- BroadcastHashJoin LeftOuter BuildRight (45) + :- Scan parquet (40) + +- BroadcastExchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -110,178 +112,186 @@ Input [2]: [c_custkey#X, count#X] Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, count#X] +Arguments: X + +(16) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [c_custkey#X, count#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [c_custkey#X, count#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(22) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) VeloxAppendBatches +(32) VeloxAppendBatches Input [2]: [c_count#X, custdist#X] Arguments: X -(31) ColumnarExchange +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(33) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(34) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(35) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(36) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(37) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(38) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(39) Scan parquet +(41) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(42) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(43) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) BroadcastExchange +(44) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(43) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(46) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) Exchange +(48) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(48) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(49) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(51) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(53) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt index 332e9d086ddf..d144dd39ca1d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt @@ -8,7 +8,7 @@ AdaptiveSparkPlan (35) +- ShuffleQueryStage (17), Statistics(X) +- ColumnarExchange (16) +- VeloxAppendBatches (15) - +- ^ RegularHashAggregateExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) @@ -86,7 +86,7 @@ Join condition: None Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(13) RegularHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt index e82d3da63e53..c4f825f5ca53 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt @@ -1,43 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (43) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- AQEShuffleRead (26) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- VeloxAppendBatches (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) + VeloxColumnarToRowExec (28) + +- AQEShuffleRead (27) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (21) :- ^ InputIteratorTransformer (7) : +- BroadcastQueryStage (5), Statistics(X) : +- ColumnarBroadcastExchange (4) : +- ^ FilterExecTransformer (2) : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- Project (39) - +- BroadcastHashJoin Inner BuildLeft (38) - :- BroadcastExchange (30) - : +- Filter (29) - : +- Scan parquet (28) - +- Filter (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (42) + +- Exchange (41) + +- Project (40) + +- BroadcastHashJoin Inner BuildLeft (39) + :- BroadcastExchange (31) + : +- Filter (30) + : +- Scan parquet (29) + +- Filter (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- Filter (33) + +- Scan parquet (32) (1) Scan parquet @@ -99,288 +100,297 @@ Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) VeloxAppendBatches +(24) VeloxAppendBatches Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(24) ColumnarExchange +(25) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) AQEShuffleRead +(27) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(27) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) Scan parquet +(29) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(29) Filter +(30) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(30) BroadcastExchange +(31) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(31) Scan parquet +(32) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(32) Filter +(33) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(33) Project +(34) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(34) HashAggregate +(35) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(35) Exchange +(36) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(37) Filter +(38) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(38) BroadcastHashJoin +(39) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(39) Project +(40) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(40) Exchange +(41) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(42) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(43) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 19 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (67) +Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ RegularHashAggregateExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ ProjectExecTransformer (47) - +- ^ FlushableHashAggregateExecTransformer (46) - +- ^ ProjectExecTransformer (45) - +- ^ FilterExecTransformer (44) - +- ^ Scan parquet (43) + VeloxColumnarToRowExec (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ ProjectExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ InputIteratorTransformer (54) + +- ShuffleQueryStage (52), Statistics(X) + +- ColumnarExchange (51) + +- VeloxAppendBatches (50) + +- ^ ProjectExecTransformer (48) + +- ^ FlushableHashAggregateExecTransformer (47) + +- ^ ProjectExecTransformer (46) + +- ^ FilterExecTransformer (45) + +- ^ Scan parquet (44) +- == Initial Plan == - HashAggregate (66) - +- HashAggregate (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (68) + +- HashAggregate (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- Filter (62) + +- Scan parquet (61) -(43) Scan parquet +(44) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(44) FilterExecTransformer +(45) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(45) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) FlushableHashAggregateExecTransformer +(47) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(48) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(49) ColumnarExchange +(50) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(51) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(51) InputAdapter +(53) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(53) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(54) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(55) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(56) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(57) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(58) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(59) Scan parquet +(61) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(60) Filter +(62) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(61) Project +(63) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(63) Exchange +(65) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(65) HashAggregate +(67) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(66) HashAggregate +(68) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(67) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt index 83440db8a3a5..741b837976a4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt @@ -1,54 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- VeloxAppendBatches (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ FilterExecTransformer (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ FilterExecTransformer (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (56) - +- Exchange (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- BroadcastHashJoin LeftAnti BuildRight (43) - : :- Filter (38) - : : +- Scan parquet (37) - : +- BroadcastExchange (42) - : +- Project (41) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Filter (45) - +- Scan parquet (44) + Sort (58) + +- Exchange (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Project (50) + +- BroadcastHashJoin Inner BuildRight (49) + :- BroadcastHashJoin LeftAnti BuildRight (45) + : :- Filter (40) + : : +- Scan parquet (39) + : +- BroadcastExchange (44) + : +- Project (43) + : +- Filter (42) + : +- Scan parquet (41) + +- BroadcastExchange (48) + +- Filter (47) + +- Scan parquet (46) (1) Scan parquet @@ -115,199 +117,207 @@ Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(16) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(25) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(26) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) VeloxAppendBatches +(31) VeloxAppendBatches Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(30) ColumnarExchange +(32) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(32) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(34) SortExecTransformer +(36) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(37) Scan parquet +(39) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(38) Filter +(40) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(39) Scan parquet +(41) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(40) Filter +(42) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(41) Project +(43) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(42) BroadcastExchange +(44) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(43) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(44) Scan parquet +(46) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(45) Filter +(47) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(46) BroadcastExchange +(48) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(47) BroadcastHashJoin +(49) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(48) Project +(50) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(49) HashAggregate +(51) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(50) Exchange +(52) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(52) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(53) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(56) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(55) Exchange +(57) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) Sort +(58) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(57) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/17.txt index a57c908f0178..1e2ed970aef4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/17.txt @@ -8,7 +8,7 @@ AdaptiveSparkPlan (36) +- ShuffleQueryStage (9), Statistics(X) +- ColumnarExchange (8) +- VeloxAppendBatches (7) - +- ^ RegularHashAggregateExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (5) +- ^ InputIteratorTransformer (4) +- RowToVeloxColumnar (2) +- LocalTableScan (1) @@ -48,7 +48,7 @@ Input [1]: [l_extendedprice#X] (4) InputIteratorTransformer Input [1]: [l_extendedprice#X] -(5) RegularHashAggregateExecTransformer +(5) FlushableHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt index a656d4e5433c..05884fea7885 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt @@ -9,7 +9,7 @@ AdaptiveSparkPlan (88) +- ColumnarExchange (48) +- VeloxAppendBatches (47) +- ^ ProjectExecTransformer (45) - +- ^ RegularHashAggregateExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (44) +- ^ ProjectExecTransformer (43) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) :- ^ ProjectExecTransformer (29) @@ -33,7 +33,7 @@ AdaptiveSparkPlan (88) : +- ColumnarExchange (15) : +- VeloxAppendBatches (14) : +- ^ ProjectExecTransformer (12) - : +- ^ RegularHashAggregateExecTransformer (11) + : +- ^ FlushableHashAggregateExecTransformer (11) : +- ^ Scan parquet (10) +- ^ InputIteratorTransformer (41) +- BroadcastQueryStage (39), Statistics(X) @@ -125,7 +125,7 @@ Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) RegularHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] @@ -261,7 +261,7 @@ Join condition: None Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(44) RegularHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt index 8cc8b37b9909..f04466031352 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt @@ -7,7 +7,7 @@ AdaptiveSparkPlan (34) +- ShuffleQueryStage (17), Statistics(X) +- ColumnarExchange (16) +- VeloxAppendBatches (15) - +- ^ RegularHashAggregateExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) @@ -85,7 +85,7 @@ Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) RegularHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt index afee0913dbde..a0edead7013d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt @@ -1,15 +1,15 @@ == Physical Plan == -AdaptiveSparkPlan (106) +AdaptiveSparkPlan (107) +- == Final Plan == - VeloxColumnarToRowExec (69) - +- AQEShuffleRead (68) - +- ShuffleQueryStage (67), Statistics(X) - +- ColumnarExchange (66) - +- VeloxAppendBatches (65) - +- ^ ProjectExecTransformer (63) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (62) - :- ^ ProjectExecTransformer (53) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (52) + VeloxColumnarToRowExec (70) + +- AQEShuffleRead (69) + +- ShuffleQueryStage (68), Statistics(X) + +- ColumnarExchange (67) + +- VeloxAppendBatches (66) + +- ^ ProjectExecTransformer (64) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (63) + :- ^ ProjectExecTransformer (54) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (53) : :- ^ InputIteratorTransformer (10) : : +- AQEShuffleRead (8) : : +- ShuffleQueryStage (7), Statistics(X) @@ -18,11 +18,11 @@ AdaptiveSparkPlan (106) : : +- ^ ProjectExecTransformer (3) : : +- ^ FilterExecTransformer (2) : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (51) - : +- BroadcastQueryStage (49), Statistics(X) - : +- ColumnarBroadcastExchange (48) - : +- ^ ProjectExecTransformer (46) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (45) + : +- ^ InputIteratorTransformer (52) + : +- BroadcastQueryStage (50), Statistics(X) + : +- ColumnarBroadcastExchange (49) + : +- ^ ProjectExecTransformer (47) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (46) : :- ^ InputIteratorTransformer (26) : : +- BroadcastQueryStage (24), Statistics(X) : : +- ColumnarBroadcastExchange (23) @@ -35,64 +35,65 @@ AdaptiveSparkPlan (106) : : +- ^ ProjectExecTransformer (15) : : +- ^ FilterExecTransformer (14) : : +- ^ Scan parquet (13) - : +- ^ FilterExecTransformer (44) - : +- ^ ProjectExecTransformer (43) - : +- ^ RegularHashAggregateExecTransformer (42) - : +- ^ InputIteratorTransformer (41) - : +- ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ FlushableHashAggregateExecTransformer (35) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (34) - : :- ^ ProjectExecTransformer (29) - : : +- ^ FilterExecTransformer (28) - : : +- ^ Scan parquet (27) - : +- ^ InputIteratorTransformer (33) - : +- BroadcastQueryStage (31), Statistics(X) - : +- ReusedExchange (30) - +- ^ InputIteratorTransformer (61) - +- BroadcastQueryStage (59), Statistics(X) - +- ColumnarBroadcastExchange (58) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ Scan parquet (54) + : +- ^ FilterExecTransformer (45) + : +- ^ ProjectExecTransformer (44) + : +- ^ RegularHashAggregateExecTransformer (43) + : +- ^ InputIteratorTransformer (42) + : +- ShuffleQueryStage (40), Statistics(X) + : +- ColumnarExchange (39) + : +- VeloxAppendBatches (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ FlushableHashAggregateExecTransformer (35) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (34) + : :- ^ ProjectExecTransformer (29) + : : +- ^ FilterExecTransformer (28) + : : +- ^ Scan parquet (27) + : +- ^ InputIteratorTransformer (33) + : +- BroadcastQueryStage (31), Statistics(X) + : +- ReusedExchange (30) + +- ^ InputIteratorTransformer (62) + +- BroadcastQueryStage (60), Statistics(X) + +- ColumnarBroadcastExchange (59) + +- ^ ProjectExecTransformer (57) + +- ^ FilterExecTransformer (56) + +- ^ Scan parquet (55) +- == Initial Plan == - Sort (105) - +- Exchange (104) - +- Project (103) - +- BroadcastHashJoin Inner BuildRight (102) - :- Project (97) - : +- ShuffledHashJoin LeftSemi BuildRight (96) - : :- Exchange (72) - : : +- Filter (71) - : : +- Scan parquet (70) - : +- Exchange (95) - : +- Project (94) - : +- BroadcastHashJoin Inner BuildLeft (93) - : :- BroadcastExchange (80) - : : +- BroadcastHashJoin LeftSemi BuildRight (79) - : : :- Filter (74) - : : : +- Scan parquet (73) - : : +- BroadcastExchange (78) - : : +- Project (77) - : : +- Filter (76) - : : +- Scan parquet (75) - : +- Filter (92) - : +- HashAggregate (91) - : +- Exchange (90) - : +- HashAggregate (89) - : +- BroadcastHashJoin LeftSemi BuildRight (88) - : :- Project (83) - : : +- Filter (82) - : : +- Scan parquet (81) - : +- BroadcastExchange (87) - : +- Project (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (101) - +- Project (100) - +- Filter (99) - +- Scan parquet (98) + Sort (106) + +- Exchange (105) + +- Project (104) + +- BroadcastHashJoin Inner BuildRight (103) + :- Project (98) + : +- ShuffledHashJoin LeftSemi BuildRight (97) + : :- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (96) + : +- Project (95) + : +- BroadcastHashJoin Inner BuildLeft (94) + : :- BroadcastExchange (81) + : : +- BroadcastHashJoin LeftSemi BuildRight (80) + : : :- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (79) + : : +- Project (78) + : : +- Filter (77) + : : +- Scan parquet (76) + : +- Filter (93) + : +- HashAggregate (92) + : +- Exchange (91) + : +- HashAggregate (90) + : +- BroadcastHashJoin LeftSemi BuildRight (89) + : :- Project (84) + : : +- Filter (83) + : : +- Scan parquet (82) + : +- BroadcastExchange (88) + : +- Project (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (102) + +- Project (101) + +- Filter (100) + +- Scan parquet (99) (1) Scan parquet @@ -251,309 +252,313 @@ Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(38) ColumnarExchange +(38) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(39) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(40) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(40) InputAdapter +(41) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(41) InputIteratorTransformer +(42) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(42) RegularHashAggregateExecTransformer +(43) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) ProjectExecTransformer +(44) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(44) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(45) BroadcastHashJoinExecTransformer +(46) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(46) ProjectExecTransformer +(47) ProjectExecTransformer Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(47) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] Arguments: false -(48) ColumnarBroadcastExchange +(49) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(49) BroadcastQueryStage +(50) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [1]: [ps_suppkey#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(52) BroadcastHashJoinExecTransformer +(53) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(53) ProjectExecTransformer +(54) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(54) Scan parquet +(55) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(55) FilterExecTransformer +(56) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(56) ProjectExecTransformer +(57) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(58) ColumnarBroadcastExchange +(59) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(59) BroadcastQueryStage +(60) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(60) InputAdapter +(61) InputAdapter Input [1]: [n_nationkey#X] -(61) InputIteratorTransformer +(62) InputIteratorTransformer Input [1]: [n_nationkey#X] -(62) BroadcastHashJoinExecTransformer +(63) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(64) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(65) VeloxAppendBatches +(66) VeloxAppendBatches Input [2]: [s_name#X, s_address#X] Arguments: X -(66) ColumnarExchange +(67) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(68) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(68) AQEShuffleRead +(69) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(69) VeloxColumnarToRowExec +(70) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(70) Scan parquet +(71) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(72) Exchange +(73) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(73) Scan parquet +(74) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(75) Scan parquet +(76) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(76) Filter +(77) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(77) Project +(78) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(78) BroadcastExchange +(79) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(79) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(80) BroadcastExchange +(81) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(81) Scan parquet +(82) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(82) Filter +(83) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(83) Project +(84) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(84) Scan parquet +(85) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(85) Filter +(86) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(86) Project +(87) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(87) BroadcastExchange +(88) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(88) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(89) HashAggregate +(90) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(90) Exchange +(91) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(92) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(92) Filter +(93) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(93) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(94) Project +(95) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(95) Exchange +(96) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(97) Project +(98) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(98) Scan parquet +(99) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(99) Filter +(100) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(100) Project +(101) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(101) BroadcastExchange +(102) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(102) BroadcastHashJoin +(103) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(103) Project +(104) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(104) Exchange +(105) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Sort +(106) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(106) AdaptiveSparkPlan +(107) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt index d34e59327a67..3ffdec004af2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt @@ -8,7 +8,7 @@ AdaptiveSparkPlan (92) +- ColumnarExchange (53) +- VeloxAppendBatches (52) +- ^ ProjectExecTransformer (50) - +- ^ RegularHashAggregateExecTransformer (49) + +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) :- ^ ProjectExecTransformer (38) @@ -286,7 +286,7 @@ Join condition: None Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(49) RegularHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt index bf6c22d0d8b8..f30752effaa4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt @@ -1,38 +1,39 @@ == Physical Plan == -AdaptiveSparkPlan (39) +AdaptiveSparkPlan (40) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- ^ SortExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- VeloxAppendBatches (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (28) + +- ^ SortExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- VeloxAppendBatches (21) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (38) - +- Exchange (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- BroadcastHashJoin LeftAnti BuildRight (32) - :- Filter (29) - : +- Scan parquet (28) - +- BroadcastExchange (31) - +- Scan parquet (30) + Sort (39) + +- Exchange (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- BroadcastHashJoin LeftAnti BuildRight (33) + :- Filter (30) + : +- Scan parquet (29) + +- BroadcastExchange (32) + +- Scan parquet (31) (1) Scan parquet @@ -94,250 +95,260 @@ Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) VeloxAppendBatches +(21) VeloxAppendBatches Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(21) ColumnarExchange +(22) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(23) InputAdapter +(24) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(25) SortExecTransformer +(26) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(26) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(28) Scan parquet +(29) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(29) Filter +(30) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(30) Scan parquet +(31) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(31) BroadcastExchange +(32) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(32) BroadcastHashJoin +(33) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(33) Project +(34) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(34) HashAggregate +(35) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(35) Exchange +(36) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(37) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(37) Exchange +(38) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Sort +(39) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(39) AdaptiveSparkPlan +(40) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (51) - +- ^ RegularHashAggregateExecTransformer (49) - +- ^ InputIteratorTransformer (48) - +- ShuffleQueryStage (46), Statistics(X) - +- ColumnarExchange (45) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (53) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ShuffleQueryStage (48), Statistics(X) + +- ColumnarExchange (47) + +- VeloxAppendBatches (46) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ FilterExecTransformer (42) + +- ^ Scan parquet (41) +- == Initial Plan == - HashAggregate (57) - +- Exchange (56) - +- HashAggregate (55) - +- Project (54) - +- Filter (53) - +- Scan parquet (52) + HashAggregate (59) + +- Exchange (58) + +- HashAggregate (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) -(40) Scan parquet +(41) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(41) FilterExecTransformer +(42) FilterExecTransformer Input [2]: [c_phone#X, c_acctbal#X] Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(43) FlushableHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(44) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(45) ColumnarExchange +(46) VeloxAppendBatches +Input [2]: [sum#X, count#X] +Arguments: X + +(47) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(46) ShuffleQueryStage +(48) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(47) InputAdapter +(49) InputAdapter Input [2]: [sum#X, count#X] -(48) InputIteratorTransformer +(50) InputIteratorTransformer Input [2]: [sum#X, count#X] -(49) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(50) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(51) VeloxColumnarToRowExec +(53) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(52) Scan parquet +(54) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(53) Filter +(55) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(54) Project +(56) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(55) HashAggregate +(57) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(56) Exchange +(58) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(57) HashAggregate +(59) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(58) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (51) - +- ^ RegularHashAggregateExecTransformer (49) - +- ^ InputIteratorTransformer (48) - +- ShuffleQueryStage (46), Statistics(X) - +- ColumnarExchange (45) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (53) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ShuffleQueryStage (48), Statistics(X) + +- ColumnarExchange (47) + +- VeloxAppendBatches (46) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ FilterExecTransformer (42) + +- ^ Scan parquet (41) +- == Initial Plan == - HashAggregate (57) - +- Exchange (56) - +- HashAggregate (55) - +- Project (54) - +- Filter (53) - +- Scan parquet (52) \ No newline at end of file + HashAggregate (59) + +- Exchange (58) + +- HashAggregate (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt index 5d022b4a9709..d9f87ca9b538 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt @@ -10,7 +10,7 @@ AdaptiveSparkPlan (54) +- ColumnarExchange (27) +- VeloxAppendBatches (26) +- ^ ProjectExecTransformer (24) - +- ^ RegularHashAggregateExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (23) +- ^ ProjectExecTransformer (22) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) :- ^ ProjectExecTransformer (12) @@ -145,7 +145,7 @@ Join condition: None Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) RegularHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt index 577a71bdeff7..db9f7716a365 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt @@ -1,44 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (46) +- == Final Plan == - VeloxColumnarToRowExec (30) - +- ^ SortExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- VeloxAppendBatches (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9), Statistics(X) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (31) + +- ^ SortExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ InputIteratorTransformer (21) + +- ShuffleQueryStage (19), Statistics(X) + +- ColumnarExchange (18) + +- VeloxAppendBatches (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- HashAggregate (42) - +- Exchange (41) - +- HashAggregate (40) - +- Project (39) - +- BroadcastHashJoin LeftSemi BuildRight (38) - :- Project (33) - : +- Filter (32) - : +- Scan parquet (31) - +- BroadcastExchange (37) - +- Project (36) - +- Filter (35) - +- Scan parquet (34) + Sort (45) + +- Exchange (44) + +- HashAggregate (43) + +- Exchange (42) + +- HashAggregate (41) + +- Project (40) + +- BroadcastHashJoin LeftSemi BuildRight (39) + :- Project (34) + : +- Filter (33) + : +- Scan parquet (32) + +- BroadcastExchange (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) (1) Scan parquet @@ -113,129 +114,133 @@ Input [2]: [o_orderpriority#X, count#X] Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(17) ColumnarExchange +(17) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(18) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(18) ShuffleQueryStage +(19) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(19) InputAdapter +(20) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(20) InputIteratorTransformer +(21) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(21) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) VeloxAppendBatches +(24) VeloxAppendBatches Input [2]: [o_orderpriority#X, order_count#X] Arguments: X -(24) ColumnarExchange +(25) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(28) SortExecTransformer +(29) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(29) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(30) VeloxColumnarToRowExec +(31) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(31) Scan parquet +(32) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(32) Filter +(33) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(33) Project +(34) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(34) Scan parquet +(35) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(35) Filter +(36) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(36) Project +(37) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(37) BroadcastExchange +(38) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(38) BroadcastHashJoin +(39) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(39) Project +(40) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(40) HashAggregate +(41) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(41) Exchange +(42) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) HashAggregate +(43) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(43) Exchange +(44) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(45) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(46) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt index 189b67f8e353..67e9d847aefa 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt @@ -1,92 +1,93 @@ == Physical Plan == -AdaptiveSparkPlan (101) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (66) - +- ^ SortExecTransformer (64) - +- ^ InputIteratorTransformer (63) - +- ShuffleQueryStage (61), Statistics(X) - +- ColumnarExchange (60) - +- VeloxAppendBatches (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45), Statistics(X) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (67) + +- ^ SortExecTransformer (65) + +- ^ InputIteratorTransformer (64) + +- ShuffleQueryStage (62), Statistics(X) + +- ColumnarExchange (61) + +- VeloxAppendBatches (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ InputIteratorTransformer (57) + +- ShuffleQueryStage (55), Statistics(X) + +- ColumnarExchange (54) + +- VeloxAppendBatches (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (100) - +- Exchange (99) - +- HashAggregate (98) - +- Exchange (97) - +- HashAggregate (96) - +- Project (95) - +- BroadcastHashJoin Inner BuildRight (94) - :- Project (89) - : +- BroadcastHashJoin Inner BuildRight (88) - : :- Project (84) - : : +- BroadcastHashJoin Inner BuildRight (83) - : : :- Project (79) - : : : +- BroadcastHashJoin Inner BuildRight (78) - : : : :- Project (74) - : : : : +- BroadcastHashJoin Inner BuildLeft (73) - : : : : :- BroadcastExchange (69) - : : : : : +- Filter (68) - : : : : : +- Scan parquet (67) - : : : : +- Project (72) - : : : : +- Filter (71) - : : : : +- Scan parquet (70) - : : : +- BroadcastExchange (77) - : : : +- Filter (76) - : : : +- Scan parquet (75) - : : +- BroadcastExchange (82) - : : +- Filter (81) - : : +- Scan parquet (80) - : +- BroadcastExchange (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (93) - +- Project (92) - +- Filter (91) - +- Scan parquet (90) + Sort (101) + +- Exchange (100) + +- HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Project (96) + +- BroadcastHashJoin Inner BuildRight (95) + :- Project (90) + : +- BroadcastHashJoin Inner BuildRight (89) + : :- Project (85) + : : +- BroadcastHashJoin Inner BuildRight (84) + : : :- Project (80) + : : : +- BroadcastHashJoin Inner BuildRight (79) + : : : :- Project (75) + : : : : +- BroadcastHashJoin Inner BuildLeft (74) + : : : : :- BroadcastExchange (70) + : : : : : +- Filter (69) + : : : : : +- Scan parquet (68) + : : : : +- Project (73) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (78) + : : : +- Filter (77) + : : : +- Scan parquet (76) + : : +- BroadcastExchange (83) + : : +- Filter (82) + : : +- Scan parquet (81) + : +- BroadcastExchange (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (94) + +- Project (93) + +- Filter (92) + +- Scan parquet (91) (1) Scan parquet @@ -313,225 +314,229 @@ Input [3]: [n_name#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(53) ColumnarExchange +(53) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(54) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(55) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(55) InputAdapter +(56) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(56) InputIteratorTransformer +(57) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(57) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(58) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(59) VeloxAppendBatches +(60) VeloxAppendBatches Input [2]: [n_name#X, revenue#X] Arguments: X -(60) ColumnarExchange +(61) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(62) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(62) InputAdapter +(63) InputAdapter Input [2]: [n_name#X, revenue#X] -(63) InputIteratorTransformer +(64) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(64) SortExecTransformer +(65) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(65) WholeStageCodegenTransformer (X) +(66) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(66) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(67) Scan parquet +(68) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(68) Filter +(69) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(69) BroadcastExchange +(70) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(70) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(72) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(74) Project +(75) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(75) Scan parquet +(76) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(76) Filter +(77) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(77) BroadcastExchange +(78) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(78) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(79) Project +(80) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(80) Scan parquet +(81) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(81) Filter +(82) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(82) BroadcastExchange +(83) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(83) BroadcastHashJoin +(84) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(84) Project +(85) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(85) Scan parquet +(86) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(87) BroadcastExchange +(88) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(89) Project +(90) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(90) Scan parquet +(91) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(91) Filter +(92) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(92) Project +(93) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(93) BroadcastExchange +(94) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(94) BroadcastHashJoin +(95) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(95) Project +(96) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(96) HashAggregate +(97) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(97) Exchange +(98) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) HashAggregate +(99) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(99) Exchange +(100) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Sort +(101) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(101) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/6.txt index 3a3148c1c113..3432579a0de0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/6.txt @@ -7,7 +7,7 @@ AdaptiveSparkPlan (20) +- ShuffleQueryStage (8), Statistics(X) +- ColumnarExchange (7) +- VeloxAppendBatches (6) - +- ^ RegularHashAggregateExecTransformer (4) + +- ^ FlushableHashAggregateExecTransformer (4) +- ^ ProjectExecTransformer (3) +- ^ FilterExecTransformer (2) +- ^ Scan parquet (1) @@ -35,7 +35,7 @@ Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnot Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) RegularHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt index 88226758f103..3db2d7669f14 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt @@ -1,86 +1,87 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (95) +- == Final Plan == - VeloxColumnarToRowExec (61) - +- ^ SortExecTransformer (59) - +- ^ InputIteratorTransformer (58) - +- ShuffleQueryStage (56), Statistics(X) - +- ColumnarExchange (55) - +- VeloxAppendBatches (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16), Statistics(X) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ FilterExecTransformer (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25), Statistics(X) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ FilterExecTransformer (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ FilterExecTransformer (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40), Statistics(X) - +- ReusedExchange (39) + VeloxColumnarToRowExec (62) + +- ^ SortExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57), Statistics(X) + +- ColumnarExchange (56) + +- VeloxAppendBatches (55) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ FilterExecTransformer (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ FilterExecTransformer (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == - Sort (93) - +- Exchange (92) - +- HashAggregate (91) - +- Exchange (90) - +- HashAggregate (89) - +- Project (88) - +- BroadcastHashJoin Inner BuildRight (87) - :- Project (83) - : +- BroadcastHashJoin Inner BuildRight (82) - : :- Project (78) - : : +- BroadcastHashJoin Inner BuildRight (77) - : : :- Project (73) - : : : +- BroadcastHashJoin Inner BuildRight (72) - : : : :- Project (68) - : : : : +- BroadcastHashJoin Inner BuildLeft (67) - : : : : :- BroadcastExchange (64) - : : : : : +- Filter (63) - : : : : : +- Scan parquet (62) - : : : : +- Filter (66) - : : : : +- Scan parquet (65) - : : : +- BroadcastExchange (71) - : : : +- Filter (70) - : : : +- Scan parquet (69) - : : +- BroadcastExchange (76) - : : +- Filter (75) - : : +- Scan parquet (74) - : +- BroadcastExchange (81) - : +- Filter (80) - : +- Scan parquet (79) - +- BroadcastExchange (86) - +- Filter (85) - +- Scan parquet (84) + Sort (94) + +- Exchange (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- BroadcastHashJoin Inner BuildRight (88) + :- Project (84) + : +- BroadcastHashJoin Inner BuildRight (83) + : :- Project (79) + : : +- BroadcastHashJoin Inner BuildRight (78) + : : :- Project (74) + : : : +- BroadcastHashJoin Inner BuildRight (73) + : : : :- Project (69) + : : : : +- BroadcastHashJoin Inner BuildLeft (68) + : : : : :- BroadcastExchange (65) + : : : : : +- Filter (64) + : : : : : +- Scan parquet (63) + : : : : +- Filter (67) + : : : : +- Scan parquet (66) + : : : +- BroadcastExchange (72) + : : : +- Filter (71) + : : : +- Scan parquet (70) + : : +- BroadcastExchange (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- BroadcastExchange (82) + : +- Filter (81) + : +- Scan parquet (80) + +- BroadcastExchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -283,217 +284,221 @@ Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(48) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(49) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(50) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(53) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) VeloxAppendBatches +(55) VeloxAppendBatches Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(55) ColumnarExchange +(56) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(56) ShuffleQueryStage +(57) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(57) InputAdapter +(58) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) InputIteratorTransformer +(59) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(59) SortExecTransformer +(60) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(60) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(61) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(62) Scan parquet +(63) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(63) Filter +(64) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(64) BroadcastExchange +(65) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(65) Scan parquet +(66) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(67) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(68) Project +(69) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(69) Scan parquet +(70) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(71) BroadcastExchange +(72) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(73) Project +(74) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(75) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(78) Project +(79) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(79) Scan parquet +(80) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(80) Filter +(81) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(81) BroadcastExchange +(82) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(83) Project +(84) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(84) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(85) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(86) BroadcastExchange +(87) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(88) Project +(89) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(89) HashAggregate +(90) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(90) Exchange +(91) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(92) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(92) Exchange +(93) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Sort +(94) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(94) AdaptiveSparkPlan +(95) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt index e0c8a43893bd..f77b5fca35c8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt @@ -1,117 +1,118 @@ == Physical Plan == -AdaptiveSparkPlan (130) +AdaptiveSparkPlan (131) +- == Final Plan == - VeloxColumnarToRowExec (85) - +- ^ SortExecTransformer (83) - +- ^ InputIteratorTransformer (82) - +- ShuffleQueryStage (80), Statistics(X) - +- ColumnarExchange (79) - +- VeloxAppendBatches (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ FilterExecTransformer (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26), Statistics(X) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ FilterExecTransformer (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35), Statistics(X) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ FilterExecTransformer (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44), Statistics(X) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ FilterExecTransformer (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53), Statistics(X) - : +- ColumnarBroadcastExchange (52) - : +- ^ FilterExecTransformer (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63), Statistics(X) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ FilterExecTransformer (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (86) + +- ^ SortExecTransformer (84) + +- ^ InputIteratorTransformer (83) + +- ShuffleQueryStage (81), Statistics(X) + +- ColumnarExchange (80) + +- VeloxAppendBatches (79) + +- ^ ProjectExecTransformer (77) + +- ^ RegularHashAggregateExecTransformer (76) + +- ^ InputIteratorTransformer (75) + +- ShuffleQueryStage (73), Statistics(X) + +- ColumnarExchange (72) + +- VeloxAppendBatches (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ FilterExecTransformer (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ FilterExecTransformer (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ FilterExecTransformer (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ FilterExecTransformer (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ FilterExecTransformer (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ FilterExecTransformer (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ FilterExecTransformer (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (129) - +- Exchange (128) - +- HashAggregate (127) - +- Exchange (126) - +- HashAggregate (125) - +- Project (124) - +- BroadcastHashJoin Inner BuildRight (123) - :- Project (118) - : +- BroadcastHashJoin Inner BuildRight (117) - : :- Project (113) - : : +- BroadcastHashJoin Inner BuildRight (112) - : : :- Project (108) - : : : +- BroadcastHashJoin Inner BuildRight (107) - : : : :- Project (103) - : : : : +- BroadcastHashJoin Inner BuildRight (102) - : : : : :- Project (98) - : : : : : +- BroadcastHashJoin Inner BuildRight (97) - : : : : : :- Project (93) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (92) - : : : : : : :- BroadcastExchange (89) - : : : : : : : +- Project (88) - : : : : : : : +- Filter (87) - : : : : : : : +- Scan parquet (86) - : : : : : : +- Filter (91) - : : : : : : +- Scan parquet (90) - : : : : : +- BroadcastExchange (96) - : : : : : +- Filter (95) - : : : : : +- Scan parquet (94) - : : : : +- BroadcastExchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- BroadcastExchange (106) - : : : +- Filter (105) - : : : +- Scan parquet (104) - : : +- BroadcastExchange (111) - : : +- Filter (110) - : : +- Scan parquet (109) - : +- BroadcastExchange (116) - : +- Filter (115) - : +- Scan parquet (114) - +- BroadcastExchange (122) - +- Project (121) - +- Filter (120) - +- Scan parquet (119) + Sort (130) + +- Exchange (129) + +- HashAggregate (128) + +- Exchange (127) + +- HashAggregate (126) + +- Project (125) + +- BroadcastHashJoin Inner BuildRight (124) + :- Project (119) + : +- BroadcastHashJoin Inner BuildRight (118) + : :- Project (114) + : : +- BroadcastHashJoin Inner BuildRight (113) + : : :- Project (109) + : : : +- BroadcastHashJoin Inner BuildRight (108) + : : : :- Project (104) + : : : : +- BroadcastHashJoin Inner BuildRight (103) + : : : : :- Project (99) + : : : : : +- BroadcastHashJoin Inner BuildRight (98) + : : : : : :- Project (94) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (93) + : : : : : : :- BroadcastExchange (90) + : : : : : : : +- Project (89) + : : : : : : : +- Filter (88) + : : : : : : : +- Scan parquet (87) + : : : : : : +- Filter (92) + : : : : : : +- Scan parquet (91) + : : : : : +- BroadcastExchange (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- BroadcastExchange (102) + : : : : +- Filter (101) + : : : : +- Scan parquet (100) + : : : +- BroadcastExchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- BroadcastExchange (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- BroadcastExchange (117) + : +- Filter (116) + : +- Scan parquet (115) + +- BroadcastExchange (123) + +- Project (122) + +- Filter (121) + +- Scan parquet (120) (1) Scan parquet @@ -414,277 +415,281 @@ Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(71) ColumnarExchange +(71) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(72) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(72) ShuffleQueryStage +(73) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(73) InputAdapter +(74) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(74) InputIteratorTransformer +(75) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(75) RegularHashAggregateExecTransformer +(76) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(76) ProjectExecTransformer +(77) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(77) WholeStageCodegenTransformer (X) +(78) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) VeloxAppendBatches +(79) VeloxAppendBatches Input [2]: [o_year#X, mkt_share#X] Arguments: X -(79) ColumnarExchange +(80) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(80) ShuffleQueryStage +(81) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(81) InputAdapter +(82) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(82) InputIteratorTransformer +(83) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(83) SortExecTransformer +(84) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(84) WholeStageCodegenTransformer (X) +(85) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(85) VeloxColumnarToRowExec +(86) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(86) Scan parquet +(87) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(87) Filter +(88) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(88) Project +(89) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(89) BroadcastExchange +(90) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(90) Scan parquet +(91) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(91) Filter +(92) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(92) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(93) Project +(94) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(94) Scan parquet +(95) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(95) Filter +(96) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(96) BroadcastExchange +(97) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(97) BroadcastHashJoin +(98) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(98) Project +(99) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(99) Scan parquet +(100) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(100) Filter +(101) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(101) BroadcastExchange +(102) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(102) BroadcastHashJoin +(103) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(103) Project +(104) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(104) Scan parquet +(105) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(105) Filter +(106) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(106) BroadcastExchange +(107) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(107) BroadcastHashJoin +(108) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(108) Project +(109) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(109) Scan parquet +(110) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(110) Filter +(111) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(111) BroadcastExchange +(112) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(112) BroadcastHashJoin +(113) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(113) Project +(114) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(114) Scan parquet +(115) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(115) Filter +(116) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(116) BroadcastExchange +(117) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(117) BroadcastHashJoin +(118) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(118) Project +(119) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(119) Scan parquet +(120) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(120) Filter +(121) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(121) Project +(122) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(122) BroadcastExchange +(123) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(123) BroadcastHashJoin +(124) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(124) Project +(125) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(125) HashAggregate +(126) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(126) Exchange +(127) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) HashAggregate +(128) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(128) Exchange +(129) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Sort +(130) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(130) AdaptiveSparkPlan +(131) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt index 75b883057963..1be487ed123a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt @@ -1,90 +1,91 @@ == Physical Plan == -AdaptiveSparkPlan (99) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- VeloxAppendBatches (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61), Statistics(X) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (98) - +- Exchange (97) - +- HashAggregate (96) - +- Exchange (95) - +- HashAggregate (94) - +- Project (93) - +- BroadcastHashJoin Inner BuildRight (92) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (69) - : : : : : +- Project (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Filter (71) - : : : : +- Scan parquet (70) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (91) - +- Filter (90) - +- Scan parquet (89) + Sort (99) + +- Exchange (98) + +- HashAggregate (97) + +- Exchange (96) + +- HashAggregate (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (70) + : : : : : +- Project (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -307,221 +308,225 @@ Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(53) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) VeloxAppendBatches +(59) VeloxAppendBatches Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(59) ColumnarExchange +(60) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(61) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(61) InputAdapter +(62) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) InputIteratorTransformer +(63) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(63) SortExecTransformer +(64) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(65) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(67) Filter +(68) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(68) Project +(69) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(69) BroadcastExchange +(70) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(70) Scan parquet +(71) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(73) Project +(74) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(74) Scan parquet +(75) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(78) Project +(79) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(79) Scan parquet +(80) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(80) Filter +(81) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(81) BroadcastExchange +(82) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(83) Project +(84) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(84) Scan parquet +(85) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(86) BroadcastExchange +(87) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(88) Project +(89) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(89) Scan parquet +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(91) BroadcastExchange +(92) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(92) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(93) Project +(94) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(94) HashAggregate +(95) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(95) Exchange +(96) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) HashAggregate +(97) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(97) Exchange +(98) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Sort +(99) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(99) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/1.txt index 468ad7c5d3aa..1e53cd90e1b3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (29) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (28) - +- Exchange (27) - +- HashAggregate (26) - +- Exchange (25) - +- HashAggregate (24) - +- Project (23) - +- Filter (22) - +- Scan parquet (21) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -57,101 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) VeloxAppendBatches +(14) VeloxAppendBatches Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) ColumnarExchange +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(18) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(20) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(21) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(22) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(23) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(24) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(25) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(27) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(29) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt index ce956550a45a..098b6610a2e1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt @@ -10,7 +10,7 @@ AdaptiveSparkPlan (68) +- ColumnarExchange (36) +- VeloxAppendBatches (35) +- ^ ProjectExecTransformer (33) - +- ^ RegularHashAggregateExecTransformer (32) + +- ^ FlushableHashAggregateExecTransformer (32) +- ^ ProjectExecTransformer (31) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) :- ^ ProjectExecTransformer (22) @@ -198,7 +198,7 @@ Join condition: None Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(32) RegularHashAggregateExecTransformer +(32) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt index 29763329fd30..725debe00d5e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt @@ -1,56 +1,57 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- ^ SortExecTransformer (37) - +- ^ InputIteratorTransformer (36) - +- ShuffleQueryStage (34), Statistics(X) - +- ColumnarExchange (33) - +- VeloxAppendBatches (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7), Statistics(X) - : +- ColumnarBroadcastExchange (6) - : +- ^ FilterExecTransformer (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17), Statistics(X) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FilterExecTransformer (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ FilterExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ FilterExecTransformer (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FilterExecTransformer (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (58) - +- Exchange (57) - +- Filter (56) - +- HashAggregate (55) - +- Exchange (54) - +- HashAggregate (53) - +- Project (52) - +- BroadcastHashJoin Inner BuildRight (51) - :- Project (46) - : +- BroadcastHashJoin Inner BuildRight (45) - : :- Filter (41) - : : +- Scan parquet (40) - : +- BroadcastExchange (44) - : +- Filter (43) - : +- Scan parquet (42) - +- BroadcastExchange (50) - +- Project (49) - +- Filter (48) - +- Scan parquet (47) + Sort (59) + +- Exchange (58) + +- Filter (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- BroadcastHashJoin Inner BuildRight (52) + :- Project (47) + : +- BroadcastHashJoin Inner BuildRight (46) + : :- Filter (42) + : : +- Scan parquet (41) + : +- BroadcastExchange (45) + : +- Filter (44) + : +- Scan parquet (43) + +- BroadcastExchange (51) + +- Project (50) + +- Filter (49) + +- Scan parquet (48) (1) Scan parquet @@ -161,389 +162,398 @@ Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(25) ColumnarExchange +(25) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(26) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(27) InputAdapter +(28) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(28) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(29) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(30) FilterExecTransformer +(31) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) VeloxAppendBatches +(33) VeloxAppendBatches Input [2]: [ps_partkey#X, value#X] Arguments: X -(33) ColumnarExchange +(34) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(34) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(35) InputAdapter +(36) InputAdapter Input [2]: [ps_partkey#X, value#X] -(36) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(37) SortExecTransformer +(38) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(38) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(39) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(40) Scan parquet +(41) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(41) Filter +(42) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(42) Scan parquet +(43) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(43) Filter +(44) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(44) BroadcastExchange +(45) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(45) BroadcastHashJoin +(46) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(46) Project +(47) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(47) Scan parquet +(48) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(48) Filter +(49) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(49) Project +(50) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(50) BroadcastExchange +(51) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(51) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(52) Project +(53) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(53) HashAggregate +(54) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(54) Exchange +(55) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) HashAggregate +(56) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(56) Filter +(57) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(57) Exchange +(58) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) Sort +(59) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(59) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 30 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (100) +Subquery:1 Hosting operator id = 31 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (83) - +- ^ ProjectExecTransformer (81) - +- ^ RegularHashAggregateExecTransformer (80) - +- ^ InputIteratorTransformer (79) - +- ShuffleQueryStage (77), Statistics(X) - +- ColumnarExchange (76) - +- ^ FlushableHashAggregateExecTransformer (74) - +- ^ ProjectExecTransformer (73) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (72) - :- ^ ProjectExecTransformer (67) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - : :- ^ FilterExecTransformer (61) - : : +- ^ Scan parquet (60) - : +- ^ InputIteratorTransformer (65) - : +- BroadcastQueryStage (63), Statistics(X) - : +- ReusedExchange (62) - +- ^ InputIteratorTransformer (71) - +- BroadcastQueryStage (69), Statistics(X) - +- ReusedExchange (68) + VeloxColumnarToRowExec (85) + +- ^ ProjectExecTransformer (83) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- VeloxAppendBatches (77) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (73) + :- ^ ProjectExecTransformer (68) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (67) + : :- ^ FilterExecTransformer (62) + : : +- ^ Scan parquet (61) + : +- ^ InputIteratorTransformer (66) + : +- BroadcastQueryStage (64), Statistics(X) + : +- ReusedExchange (63) + +- ^ InputIteratorTransformer (72) + +- BroadcastQueryStage (70), Statistics(X) + +- ReusedExchange (69) +- == Initial Plan == - HashAggregate (99) - +- Exchange (98) - +- HashAggregate (97) - +- Project (96) - +- BroadcastHashJoin Inner BuildRight (95) - :- Project (90) - : +- BroadcastHashJoin Inner BuildRight (89) - : :- Filter (85) - : : +- Scan parquet (84) - : +- BroadcastExchange (88) - : +- Filter (87) - : +- Scan parquet (86) - +- BroadcastExchange (94) - +- Project (93) - +- Filter (92) - +- Scan parquet (91) - - -(60) Scan parquet + HashAggregate (101) + +- Exchange (100) + +- HashAggregate (99) + +- Project (98) + +- BroadcastHashJoin Inner BuildRight (97) + :- Project (92) + : +- BroadcastHashJoin Inner BuildRight (91) + : :- Filter (87) + : : +- Scan parquet (86) + : +- BroadcastExchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- BroadcastExchange (96) + +- Project (95) + +- Filter (94) + +- Scan parquet (93) + + +(61) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(61) FilterExecTransformer +(62) FilterExecTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: isnotnull(ps_suppkey#X) -(62) ReusedExchange [Reuses operator id: 6] +(63) ReusedExchange [Reuses operator id: 6] Output [2]: [s_suppkey#X, s_nationkey#X] -(63) BroadcastQueryStage +(64) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(64) InputAdapter +(65) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(65) InputIteratorTransformer +(66) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(66) BroadcastHashJoinExecTransformer +(67) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(67) ProjectExecTransformer +(68) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(68) ReusedExchange [Reuses operator id: 16] +(69) ReusedExchange [Reuses operator id: 16] Output [1]: [n_nationkey#X] -(69) BroadcastQueryStage +(70) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(70) InputAdapter +(71) InputAdapter Input [1]: [n_nationkey#X] -(71) InputIteratorTransformer +(72) InputIteratorTransformer Input [1]: [n_nationkey#X] -(72) BroadcastHashJoinExecTransformer +(73) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(73) ProjectExecTransformer +(74) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(74) FlushableHashAggregateExecTransformer +(75) FlushableHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(75) WholeStageCodegenTransformer (X) +(76) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(76) ColumnarExchange +(77) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(78) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(77) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(78) InputAdapter +(80) InputAdapter Input [2]: [sum#X, isEmpty#X] -(79) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(80) RegularHashAggregateExecTransformer +(82) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(81) ProjectExecTransformer +(83) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(82) WholeStageCodegenTransformer (X) +(84) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(83) VeloxColumnarToRowExec +(85) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(84) Scan parquet +(86) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(85) Filter +(87) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(86) Scan parquet +(88) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(87) Filter +(89) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(88) BroadcastExchange +(90) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(89) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(90) Project +(92) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(91) Scan parquet +(93) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(92) Filter +(94) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(93) Project +(95) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(94) BroadcastExchange +(96) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(95) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(96) Project +(98) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(97) HashAggregate +(99) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(98) Exchange +(100) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(99) HashAggregate +(101) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(100) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt index 8b6161e1f224..e2b9e5153009 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt @@ -1,42 +1,43 @@ == Physical Plan == -AdaptiveSparkPlan (43) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- VeloxAppendBatches (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (42) - +- Exchange (41) - +- HashAggregate (40) - +- Exchange (39) - +- HashAggregate (38) - +- Project (37) - +- BroadcastHashJoin Inner BuildLeft (36) - :- BroadcastExchange (32) - : +- Filter (31) - : +- Scan parquet (30) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (43) + +- Exchange (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- BroadcastHashJoin Inner BuildLeft (37) + :- BroadcastExchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -108,126 +109,130 @@ Input [3]: [l_shipmode#X, sum#X, sum#X] Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(16) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(20) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) VeloxAppendBatches +(23) VeloxAppendBatches Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(23) ColumnarExchange +(24) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(27) SortExecTransformer +(28) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(30) Scan parquet +(31) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(32) BroadcastExchange +(33) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(33) Scan parquet +(34) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(34) Filter +(35) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(35) Project +(36) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(36) BroadcastHashJoin +(37) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(37) Project +(38) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(38) HashAggregate +(39) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(39) Exchange +(40) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) HashAggregate +(41) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(41) Exchange +(42) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Sort +(43) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(43) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt index f84ec67ca125..f64de4dee4b2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt @@ -1,50 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32), Statistics(X) - +- ColumnarExchange (31) - +- VeloxAppendBatches (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FilterExecTransformer (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ FilterExecTransformer (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (52) - +- Exchange (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- BroadcastHashJoin LeftOuter BuildRight (43) - :- Scan parquet (38) - +- BroadcastExchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- BroadcastHashJoin LeftOuter BuildRight (45) + :- Scan parquet (40) + +- BroadcastExchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -111,179 +113,187 @@ Input [2]: [c_custkey#X, count#X] Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, count#X] +Arguments: X + +(16) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [c_custkey#X, count#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [c_custkey#X, count#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(22) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) VeloxAppendBatches +(32) VeloxAppendBatches Input [2]: [c_count#X, custdist#X] Arguments: X -(31) ColumnarExchange +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(33) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(34) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(35) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(36) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(37) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(38) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(39) Scan parquet +(41) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(42) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(43) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) BroadcastExchange +(44) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(43) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(44) Project +(46) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) Exchange +(48) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(48) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(49) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(51) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(53) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt index ef33284c5ef7..901c481fac69 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt @@ -8,7 +8,7 @@ AdaptiveSparkPlan (35) +- ShuffleQueryStage (17), Statistics(X) +- ColumnarExchange (16) +- VeloxAppendBatches (15) - +- ^ RegularHashAggregateExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) @@ -87,7 +87,7 @@ Join condition: None Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(13) RegularHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt index 978722c255d7..a87fdff4537d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt @@ -1,43 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (43) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- AQEShuffleRead (26) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- VeloxAppendBatches (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) + VeloxColumnarToRowExec (28) + +- AQEShuffleRead (27) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (21) :- ^ InputIteratorTransformer (7) : +- BroadcastQueryStage (5), Statistics(X) : +- ColumnarBroadcastExchange (4) : +- ^ FilterExecTransformer (2) : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- Project (39) - +- BroadcastHashJoin Inner BuildLeft (38) - :- BroadcastExchange (30) - : +- Filter (29) - : +- Scan parquet (28) - +- Filter (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (42) + +- Exchange (41) + +- Project (40) + +- BroadcastHashJoin Inner BuildLeft (39) + :- BroadcastExchange (31) + : +- Filter (30) + : +- Scan parquet (29) + +- Filter (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- Filter (33) + +- Scan parquet (32) (1) Scan parquet @@ -99,290 +100,299 @@ Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) VeloxAppendBatches +(24) VeloxAppendBatches Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(24) ColumnarExchange +(25) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) AQEShuffleRead +(27) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(27) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) Scan parquet +(29) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(29) Filter +(30) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(30) BroadcastExchange +(31) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(31) Scan parquet +(32) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(32) Filter +(33) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(33) Project +(34) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(34) HashAggregate +(35) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(35) Exchange +(36) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(37) Filter +(38) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(38) BroadcastHashJoin +(39) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(39) Project +(40) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(40) Exchange +(41) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(42) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(43) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 19 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (67) +Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ RegularHashAggregateExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ ProjectExecTransformer (47) - +- ^ FlushableHashAggregateExecTransformer (46) - +- ^ ProjectExecTransformer (45) - +- ^ FilterExecTransformer (44) - +- ^ Scan parquet (43) + VeloxColumnarToRowExec (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ ProjectExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ InputIteratorTransformer (54) + +- ShuffleQueryStage (52), Statistics(X) + +- ColumnarExchange (51) + +- VeloxAppendBatches (50) + +- ^ ProjectExecTransformer (48) + +- ^ FlushableHashAggregateExecTransformer (47) + +- ^ ProjectExecTransformer (46) + +- ^ FilterExecTransformer (45) + +- ^ Scan parquet (44) +- == Initial Plan == - HashAggregate (66) - +- HashAggregate (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (68) + +- HashAggregate (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- Filter (62) + +- Scan parquet (61) -(43) Scan parquet +(44) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(44) FilterExecTransformer +(45) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(45) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) FlushableHashAggregateExecTransformer +(47) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(48) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(49) ColumnarExchange +(50) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(51) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(51) InputAdapter +(53) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(53) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(54) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(55) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(56) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(57) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(58) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(59) Scan parquet +(61) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(60) Filter +(62) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(61) Project +(63) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(63) Exchange +(65) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(65) HashAggregate +(67) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(66) HashAggregate +(68) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(67) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt index fe411808b790..e2f073b8908f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt @@ -1,54 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- VeloxAppendBatches (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ FilterExecTransformer (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ FilterExecTransformer (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (56) - +- Exchange (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- BroadcastHashJoin LeftAnti BuildRight (43) - : :- Filter (38) - : : +- Scan parquet (37) - : +- BroadcastExchange (42) - : +- Project (41) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Filter (45) - +- Scan parquet (44) + Sort (58) + +- Exchange (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Project (50) + +- BroadcastHashJoin Inner BuildRight (49) + :- BroadcastHashJoin LeftAnti BuildRight (45) + : :- Filter (40) + : : +- Scan parquet (39) + : +- BroadcastExchange (44) + : +- Project (43) + : +- Filter (42) + : +- Scan parquet (41) + +- BroadcastExchange (48) + +- Filter (47) + +- Scan parquet (46) (1) Scan parquet @@ -116,201 +118,209 @@ Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(16) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(25) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(26) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) VeloxAppendBatches +(31) VeloxAppendBatches Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(30) ColumnarExchange +(32) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(32) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(34) SortExecTransformer +(36) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(37) Scan parquet +(39) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(38) Filter +(40) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(39) Scan parquet +(41) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(40) Filter +(42) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(41) Project +(43) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(42) BroadcastExchange +(44) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(43) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: LeftAnti Join condition: None -(44) Scan parquet +(46) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(45) Filter +(47) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(46) BroadcastExchange +(48) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(47) BroadcastHashJoin +(49) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(48) Project +(50) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(49) HashAggregate +(51) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(50) Exchange +(52) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(52) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(53) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(56) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(55) Exchange +(57) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) Sort +(58) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(57) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/17.txt index e499293269fc..77733ad6f8a5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/17.txt @@ -8,7 +8,7 @@ AdaptiveSparkPlan (36) +- ShuffleQueryStage (9), Statistics(X) +- ColumnarExchange (8) +- VeloxAppendBatches (7) - +- ^ RegularHashAggregateExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (5) +- ^ InputIteratorTransformer (4) +- RowToVeloxColumnar (2) +- LocalTableScan (1) @@ -48,7 +48,7 @@ Input [1]: [l_extendedprice#X] (4) InputIteratorTransformer Input [1]: [l_extendedprice#X] -(5) RegularHashAggregateExecTransformer +(5) FlushableHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt index 8899b0c255ae..4f36a185da72 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt @@ -9,7 +9,7 @@ AdaptiveSparkPlan (88) +- ColumnarExchange (48) +- VeloxAppendBatches (47) +- ^ ProjectExecTransformer (45) - +- ^ RegularHashAggregateExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (44) +- ^ ProjectExecTransformer (43) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) :- ^ ProjectExecTransformer (29) @@ -33,7 +33,7 @@ AdaptiveSparkPlan (88) : +- ColumnarExchange (15) : +- VeloxAppendBatches (14) : +- ^ ProjectExecTransformer (12) - : +- ^ RegularHashAggregateExecTransformer (11) + : +- ^ FlushableHashAggregateExecTransformer (11) : +- ^ Scan parquet (10) +- ^ InputIteratorTransformer (41) +- BroadcastQueryStage (39), Statistics(X) @@ -125,7 +125,7 @@ Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) RegularHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] @@ -265,7 +265,7 @@ Join condition: None Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(44) RegularHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt index 0373360159eb..7854a767b26b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt @@ -7,7 +7,7 @@ AdaptiveSparkPlan (34) +- ShuffleQueryStage (17), Statistics(X) +- ColumnarExchange (16) +- VeloxAppendBatches (15) - +- ^ RegularHashAggregateExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) @@ -86,7 +86,7 @@ Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) RegularHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt index 8a1415a5bca2..981017da501e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt @@ -1,22 +1,22 @@ == Physical Plan == -AdaptiveSparkPlan (97) +AdaptiveSparkPlan (98) +- == Final Plan == - VeloxColumnarToRowExec (61) - +- AQEShuffleRead (60) - +- ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- VeloxAppendBatches (57) - +- ^ ProjectExecTransformer (55) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (54) - :- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (44) + VeloxColumnarToRowExec (62) + +- AQEShuffleRead (61) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ ProjectExecTransformer (56) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (55) + :- ^ ProjectExecTransformer (46) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (45) : :- ^ FilterExecTransformer (2) : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (43) - : +- BroadcastQueryStage (41), Statistics(X) - : +- ColumnarBroadcastExchange (40) - : +- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (37) + : +- ^ InputIteratorTransformer (44) + : +- BroadcastQueryStage (42), Statistics(X) + : +- ColumnarBroadcastExchange (41) + : +- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (38) : :- ^ InputIteratorTransformer (18) : : +- BroadcastQueryStage (16), Statistics(X) : : +- ColumnarBroadcastExchange (15) @@ -29,63 +29,64 @@ AdaptiveSparkPlan (97) : : +- ^ ProjectExecTransformer (7) : : +- ^ FilterExecTransformer (6) : : +- ^ Scan parquet (5) - : +- ^ FilterExecTransformer (36) - : +- ^ ProjectExecTransformer (35) - : +- ^ RegularHashAggregateExecTransformer (34) - : +- ^ InputIteratorTransformer (33) - : +- ShuffleQueryStage (31), Statistics(X) - : +- ColumnarExchange (30) - : +- ^ ProjectExecTransformer (28) - : +- ^ FlushableHashAggregateExecTransformer (27) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ ProjectExecTransformer (21) - : : +- ^ FilterExecTransformer (20) - : : +- ^ Scan parquet (19) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23), Statistics(X) - : +- ReusedExchange (22) - +- ^ InputIteratorTransformer (53) - +- BroadcastQueryStage (51), Statistics(X) - +- ColumnarBroadcastExchange (50) - +- ^ ProjectExecTransformer (48) - +- ^ FilterExecTransformer (47) - +- ^ Scan parquet (46) + : +- ^ FilterExecTransformer (37) + : +- ^ ProjectExecTransformer (36) + : +- ^ RegularHashAggregateExecTransformer (35) + : +- ^ InputIteratorTransformer (34) + : +- ShuffleQueryStage (32), Statistics(X) + : +- ColumnarExchange (31) + : +- VeloxAppendBatches (30) + : +- ^ ProjectExecTransformer (28) + : +- ^ FlushableHashAggregateExecTransformer (27) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) + : :- ^ ProjectExecTransformer (21) + : : +- ^ FilterExecTransformer (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (25) + : +- BroadcastQueryStage (23), Statistics(X) + : +- ReusedExchange (22) + +- ^ InputIteratorTransformer (54) + +- BroadcastQueryStage (52), Statistics(X) + +- ColumnarBroadcastExchange (51) + +- ^ ProjectExecTransformer (49) + +- ^ FilterExecTransformer (48) + +- ^ Scan parquet (47) +- == Initial Plan == - Sort (96) - +- Exchange (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (88) - : +- BroadcastHashJoin LeftSemi BuildRight (87) - : :- Filter (63) - : : +- Scan parquet (62) - : +- BroadcastExchange (86) - : +- Project (85) - : +- BroadcastHashJoin Inner BuildLeft (84) - : :- BroadcastExchange (71) - : : +- BroadcastHashJoin LeftSemi BuildRight (70) - : : :- Filter (65) - : : : +- Scan parquet (64) - : : +- BroadcastExchange (69) - : : +- Project (68) - : : +- Filter (67) - : : +- Scan parquet (66) - : +- Filter (83) - : +- HashAggregate (82) - : +- Exchange (81) - : +- HashAggregate (80) - : +- BroadcastHashJoin LeftSemi BuildRight (79) - : :- Project (74) - : : +- Filter (73) - : : +- Scan parquet (72) - : +- BroadcastExchange (78) - : +- Project (77) - : +- Filter (76) - : +- Scan parquet (75) - +- BroadcastExchange (92) - +- Project (91) - +- Filter (90) - +- Scan parquet (89) + Sort (97) + +- Exchange (96) + +- Project (95) + +- BroadcastHashJoin Inner BuildRight (94) + :- Project (89) + : +- BroadcastHashJoin LeftSemi BuildRight (88) + : :- Filter (64) + : : +- Scan parquet (63) + : +- BroadcastExchange (87) + : +- Project (86) + : +- BroadcastHashJoin Inner BuildLeft (85) + : :- BroadcastExchange (72) + : : +- BroadcastHashJoin LeftSemi BuildRight (71) + : : :- Filter (66) + : : : +- Scan parquet (65) + : : +- BroadcastExchange (70) + : : +- Project (69) + : : +- Filter (68) + : : +- Scan parquet (67) + : +- Filter (84) + : +- HashAggregate (83) + : +- Exchange (82) + : +- HashAggregate (81) + : +- BroadcastHashJoin LeftSemi BuildRight (80) + : :- Project (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- BroadcastExchange (79) + : +- Project (78) + : +- Filter (77) + : +- Scan parquet (76) + +- BroadcastExchange (93) + +- Project (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -216,313 +217,317 @@ Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(30) ColumnarExchange +(30) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(31) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(32) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(32) InputAdapter +(33) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(33) InputIteratorTransformer +(34) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(35) ProjectExecTransformer +(36) ProjectExecTransformer Output [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(36) FilterExecTransformer +(37) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(37) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(38) ProjectExecTransformer +(39) ProjectExecTransformer Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(39) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] Arguments: false -(40) ColumnarBroadcastExchange +(41) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(41) BroadcastQueryStage +(42) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(42) InputAdapter +(43) InputAdapter Input [1]: [ps_suppkey#X] -(43) InputIteratorTransformer +(44) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(44) BroadcastHashJoinExecTransformer +(45) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(45) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(46) Scan parquet +(47) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(47) FilterExecTransformer +(48) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(48) ProjectExecTransformer +(49) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(50) ColumnarBroadcastExchange +(51) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(51) BroadcastQueryStage +(52) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(52) InputAdapter +(53) InputAdapter Input [1]: [n_nationkey#X] -(53) InputIteratorTransformer +(54) InputIteratorTransformer Input [1]: [n_nationkey#X] -(54) BroadcastHashJoinExecTransformer +(55) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(55) ProjectExecTransformer +(56) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(56) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(57) VeloxAppendBatches +(58) VeloxAppendBatches Input [2]: [s_name#X, s_address#X] Arguments: X -(58) ColumnarExchange +(59) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(60) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(60) AQEShuffleRead +(61) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(61) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(62) Scan parquet +(63) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(63) Filter +(64) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(64) Scan parquet +(65) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(65) Filter +(66) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(66) Scan parquet +(67) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(67) Filter +(68) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(68) Project +(69) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(69) BroadcastExchange +(70) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(70) BroadcastHashJoin +(71) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(71) BroadcastExchange +(72) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(72) Scan parquet +(73) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(73) Filter +(74) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(74) Project +(75) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(75) Scan parquet +(76) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(76) Filter +(77) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(77) Project +(78) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(78) BroadcastExchange +(79) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(79) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(80) HashAggregate +(81) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(81) Exchange +(82) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) HashAggregate +(83) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(83) Filter +(84) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(84) BroadcastHashJoin +(85) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(85) Project +(86) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(86) BroadcastExchange +(87) BroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(88) Project +(89) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(89) Scan parquet +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(91) Project +(92) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(92) BroadcastExchange +(93) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(93) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(94) Project +(95) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(95) Exchange +(96) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) Sort +(97) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(97) AdaptiveSparkPlan +(98) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt index 48fc651f66f1..5aef62e33765 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt @@ -8,7 +8,7 @@ AdaptiveSparkPlan (92) +- ColumnarExchange (53) +- VeloxAppendBatches (52) +- ^ ProjectExecTransformer (50) - +- ^ RegularHashAggregateExecTransformer (49) + +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) :- ^ ProjectExecTransformer (38) @@ -291,7 +291,7 @@ Join condition: None Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(49) RegularHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt index b740990ca780..fbda7224d642 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt @@ -1,38 +1,39 @@ == Physical Plan == -AdaptiveSparkPlan (39) +AdaptiveSparkPlan (40) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- ^ SortExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- VeloxAppendBatches (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (28) + +- ^ SortExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- VeloxAppendBatches (21) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (38) - +- Exchange (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- BroadcastHashJoin LeftAnti BuildRight (32) - :- Filter (29) - : +- Scan parquet (28) - +- BroadcastExchange (31) - +- Scan parquet (30) + Sort (39) + +- Exchange (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- BroadcastHashJoin LeftAnti BuildRight (33) + :- Filter (30) + : +- Scan parquet (29) + +- BroadcastExchange (32) + +- Scan parquet (31) (1) Scan parquet @@ -95,251 +96,261 @@ Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) VeloxAppendBatches +(21) VeloxAppendBatches Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(21) ColumnarExchange +(22) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(23) InputAdapter +(24) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(25) SortExecTransformer +(26) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(26) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(28) Scan parquet +(29) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(29) Filter +(30) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(30) Scan parquet +(31) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(31) BroadcastExchange +(32) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(32) BroadcastHashJoin +(33) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(33) Project +(34) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(34) HashAggregate +(35) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(35) Exchange +(36) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(37) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(37) Exchange +(38) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Sort +(39) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(39) AdaptiveSparkPlan +(40) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (51) - +- ^ RegularHashAggregateExecTransformer (49) - +- ^ InputIteratorTransformer (48) - +- ShuffleQueryStage (46), Statistics(X) - +- ColumnarExchange (45) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (53) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ShuffleQueryStage (48), Statistics(X) + +- ColumnarExchange (47) + +- VeloxAppendBatches (46) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ FilterExecTransformer (42) + +- ^ Scan parquet (41) +- == Initial Plan == - HashAggregate (57) - +- Exchange (56) - +- HashAggregate (55) - +- Project (54) - +- Filter (53) - +- Scan parquet (52) + HashAggregate (59) + +- Exchange (58) + +- HashAggregate (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) -(40) Scan parquet +(41) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(41) FilterExecTransformer +(42) FilterExecTransformer Input [2]: [c_phone#X, c_acctbal#X] Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(43) FlushableHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(44) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(45) ColumnarExchange +(46) VeloxAppendBatches +Input [2]: [sum#X, count#X] +Arguments: X + +(47) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(46) ShuffleQueryStage +(48) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(47) InputAdapter +(49) InputAdapter Input [2]: [sum#X, count#X] -(48) InputIteratorTransformer +(50) InputIteratorTransformer Input [2]: [sum#X, count#X] -(49) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(50) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(51) VeloxColumnarToRowExec +(53) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(52) Scan parquet +(54) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(53) Filter +(55) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(54) Project +(56) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(55) HashAggregate +(57) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(56) Exchange +(58) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(57) HashAggregate +(59) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(58) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (51) - +- ^ RegularHashAggregateExecTransformer (49) - +- ^ InputIteratorTransformer (48) - +- ShuffleQueryStage (46), Statistics(X) - +- ColumnarExchange (45) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (53) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ShuffleQueryStage (48), Statistics(X) + +- ColumnarExchange (47) + +- VeloxAppendBatches (46) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ FilterExecTransformer (42) + +- ^ Scan parquet (41) +- == Initial Plan == - HashAggregate (57) - +- Exchange (56) - +- HashAggregate (55) - +- Project (54) - +- Filter (53) - +- Scan parquet (52) \ No newline at end of file + HashAggregate (59) + +- Exchange (58) + +- HashAggregate (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt index 5eb8568582af..6d518ac27214 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt @@ -10,7 +10,7 @@ AdaptiveSparkPlan (54) +- ColumnarExchange (27) +- VeloxAppendBatches (26) +- ^ ProjectExecTransformer (24) - +- ^ RegularHashAggregateExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (23) +- ^ ProjectExecTransformer (22) +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) :- ^ ProjectExecTransformer (12) @@ -147,7 +147,7 @@ Join condition: None Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) RegularHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt index 7c123f45309b..bb6c149c39e1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt @@ -1,44 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (46) +- == Final Plan == - VeloxColumnarToRowExec (30) - +- ^ SortExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- VeloxAppendBatches (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9), Statistics(X) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (31) + +- ^ SortExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ InputIteratorTransformer (21) + +- ShuffleQueryStage (19), Statistics(X) + +- ColumnarExchange (18) + +- VeloxAppendBatches (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- HashAggregate (42) - +- Exchange (41) - +- HashAggregate (40) - +- Project (39) - +- BroadcastHashJoin LeftSemi BuildRight (38) - :- Project (33) - : +- Filter (32) - : +- Scan parquet (31) - +- BroadcastExchange (37) - +- Project (36) - +- Filter (35) - +- Scan parquet (34) + Sort (45) + +- Exchange (44) + +- HashAggregate (43) + +- Exchange (42) + +- HashAggregate (41) + +- Project (40) + +- BroadcastHashJoin LeftSemi BuildRight (39) + :- Project (34) + : +- Filter (33) + : +- Scan parquet (32) + +- BroadcastExchange (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) (1) Scan parquet @@ -114,130 +115,134 @@ Input [2]: [o_orderpriority#X, count#X] Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(17) ColumnarExchange +(17) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(18) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(18) ShuffleQueryStage +(19) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(19) InputAdapter +(20) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(20) InputIteratorTransformer +(21) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(21) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) VeloxAppendBatches +(24) VeloxAppendBatches Input [2]: [o_orderpriority#X, order_count#X] Arguments: X -(24) ColumnarExchange +(25) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(28) SortExecTransformer +(29) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(29) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(30) VeloxColumnarToRowExec +(31) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(31) Scan parquet +(32) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(32) Filter +(33) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(33) Project +(34) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(34) Scan parquet +(35) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(35) Filter +(36) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(36) Project +(37) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(37) BroadcastExchange +(38) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(38) BroadcastHashJoin +(39) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(39) Project +(40) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(40) HashAggregate +(41) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(41) Exchange +(42) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) HashAggregate +(43) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(43) Exchange +(44) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(45) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(46) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt index d2b7ce10fb33..afb9aa369966 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt @@ -1,92 +1,93 @@ == Physical Plan == -AdaptiveSparkPlan (101) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (66) - +- ^ SortExecTransformer (64) - +- ^ InputIteratorTransformer (63) - +- ShuffleQueryStage (61), Statistics(X) - +- ColumnarExchange (60) - +- VeloxAppendBatches (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45), Statistics(X) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (67) + +- ^ SortExecTransformer (65) + +- ^ InputIteratorTransformer (64) + +- ShuffleQueryStage (62), Statistics(X) + +- ColumnarExchange (61) + +- VeloxAppendBatches (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ InputIteratorTransformer (57) + +- ShuffleQueryStage (55), Statistics(X) + +- ColumnarExchange (54) + +- VeloxAppendBatches (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (100) - +- Exchange (99) - +- HashAggregate (98) - +- Exchange (97) - +- HashAggregate (96) - +- Project (95) - +- BroadcastHashJoin Inner BuildRight (94) - :- Project (89) - : +- BroadcastHashJoin Inner BuildRight (88) - : :- Project (84) - : : +- BroadcastHashJoin Inner BuildRight (83) - : : :- Project (79) - : : : +- BroadcastHashJoin Inner BuildRight (78) - : : : :- Project (74) - : : : : +- BroadcastHashJoin Inner BuildLeft (73) - : : : : :- BroadcastExchange (69) - : : : : : +- Filter (68) - : : : : : +- Scan parquet (67) - : : : : +- Project (72) - : : : : +- Filter (71) - : : : : +- Scan parquet (70) - : : : +- BroadcastExchange (77) - : : : +- Filter (76) - : : : +- Scan parquet (75) - : : +- BroadcastExchange (82) - : : +- Filter (81) - : : +- Scan parquet (80) - : +- BroadcastExchange (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (93) - +- Project (92) - +- Filter (91) - +- Scan parquet (90) + Sort (101) + +- Exchange (100) + +- HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Project (96) + +- BroadcastHashJoin Inner BuildRight (95) + :- Project (90) + : +- BroadcastHashJoin Inner BuildRight (89) + : :- Project (85) + : : +- BroadcastHashJoin Inner BuildRight (84) + : : :- Project (80) + : : : +- BroadcastHashJoin Inner BuildRight (79) + : : : :- Project (75) + : : : : +- BroadcastHashJoin Inner BuildLeft (74) + : : : : :- BroadcastExchange (70) + : : : : : +- Filter (69) + : : : : : +- Scan parquet (68) + : : : : +- Project (73) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (78) + : : : +- Filter (77) + : : : +- Scan parquet (76) + : : +- BroadcastExchange (83) + : : +- Filter (82) + : : +- Scan parquet (81) + : +- BroadcastExchange (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (94) + +- Project (93) + +- Filter (92) + +- Scan parquet (91) (1) Scan parquet @@ -318,230 +319,234 @@ Input [3]: [n_name#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(53) ColumnarExchange +(53) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(54) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(55) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(55) InputAdapter +(56) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(56) InputIteratorTransformer +(57) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(57) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(58) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(59) VeloxAppendBatches +(60) VeloxAppendBatches Input [2]: [n_name#X, revenue#X] Arguments: X -(60) ColumnarExchange +(61) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(62) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(62) InputAdapter +(63) InputAdapter Input [2]: [n_name#X, revenue#X] -(63) InputIteratorTransformer +(64) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(64) SortExecTransformer +(65) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(65) WholeStageCodegenTransformer (X) +(66) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(66) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(67) Scan parquet +(68) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(68) Filter +(69) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(69) BroadcastExchange +(70) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(70) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(72) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(74) Project +(75) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(75) Scan parquet +(76) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(76) Filter +(77) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(77) BroadcastExchange +(78) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(78) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(79) Project +(80) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(80) Scan parquet +(81) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(81) Filter +(82) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(82) BroadcastExchange +(83) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(83) BroadcastHashJoin +(84) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(84) Project +(85) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(85) Scan parquet +(86) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(87) BroadcastExchange +(88) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(89) Project +(90) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(90) Scan parquet +(91) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(91) Filter +(92) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(92) Project +(93) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(93) BroadcastExchange +(94) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(94) BroadcastHashJoin +(95) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(95) Project +(96) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(96) HashAggregate +(97) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(97) Exchange +(98) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) HashAggregate +(99) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(99) Exchange +(100) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Sort +(101) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(101) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/6.txt index 415d85f46ee5..ddc921e22d0f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/6.txt @@ -7,7 +7,7 @@ AdaptiveSparkPlan (20) +- ShuffleQueryStage (8), Statistics(X) +- ColumnarExchange (7) +- VeloxAppendBatches (6) - +- ^ RegularHashAggregateExecTransformer (4) + +- ^ FlushableHashAggregateExecTransformer (4) +- ^ ProjectExecTransformer (3) +- ^ FilterExecTransformer (2) +- ^ Scan parquet (1) @@ -35,7 +35,7 @@ Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnot Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * l_discount#X) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) RegularHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt index 311144d09ec9..5ca9bbe39ef2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt @@ -1,86 +1,87 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (95) +- == Final Plan == - VeloxColumnarToRowExec (61) - +- ^ SortExecTransformer (59) - +- ^ InputIteratorTransformer (58) - +- ShuffleQueryStage (56), Statistics(X) - +- ColumnarExchange (55) - +- VeloxAppendBatches (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16), Statistics(X) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ FilterExecTransformer (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25), Statistics(X) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ FilterExecTransformer (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ FilterExecTransformer (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40), Statistics(X) - +- ReusedExchange (39) + VeloxColumnarToRowExec (62) + +- ^ SortExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57), Statistics(X) + +- ColumnarExchange (56) + +- VeloxAppendBatches (55) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ FilterExecTransformer (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ FilterExecTransformer (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == - Sort (93) - +- Exchange (92) - +- HashAggregate (91) - +- Exchange (90) - +- HashAggregate (89) - +- Project (88) - +- BroadcastHashJoin Inner BuildRight (87) - :- Project (83) - : +- BroadcastHashJoin Inner BuildRight (82) - : :- Project (78) - : : +- BroadcastHashJoin Inner BuildRight (77) - : : :- Project (73) - : : : +- BroadcastHashJoin Inner BuildRight (72) - : : : :- Project (68) - : : : : +- BroadcastHashJoin Inner BuildLeft (67) - : : : : :- BroadcastExchange (64) - : : : : : +- Filter (63) - : : : : : +- Scan parquet (62) - : : : : +- Filter (66) - : : : : +- Scan parquet (65) - : : : +- BroadcastExchange (71) - : : : +- Filter (70) - : : : +- Scan parquet (69) - : : +- BroadcastExchange (76) - : : +- Filter (75) - : : +- Scan parquet (74) - : +- BroadcastExchange (81) - : +- Filter (80) - : +- Scan parquet (79) - +- BroadcastExchange (86) - +- Filter (85) - +- Scan parquet (84) + Sort (94) + +- Exchange (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- BroadcastHashJoin Inner BuildRight (88) + :- Project (84) + : +- BroadcastHashJoin Inner BuildRight (83) + : :- Project (79) + : : +- BroadcastHashJoin Inner BuildRight (78) + : : :- Project (74) + : : : +- BroadcastHashJoin Inner BuildRight (73) + : : : :- Project (69) + : : : : +- BroadcastHashJoin Inner BuildLeft (68) + : : : : :- BroadcastExchange (65) + : : : : : +- Filter (64) + : : : : : +- Scan parquet (63) + : : : : +- Filter (67) + : : : : +- Scan parquet (66) + : : : +- BroadcastExchange (72) + : : : +- Filter (71) + : : : +- Scan parquet (70) + : : +- BroadcastExchange (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- BroadcastExchange (82) + : +- Filter (81) + : +- Scan parquet (80) + +- BroadcastExchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -288,222 +289,226 @@ Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(48) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(49) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(50) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(53) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) VeloxAppendBatches +(55) VeloxAppendBatches Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(55) ColumnarExchange +(56) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(56) ShuffleQueryStage +(57) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(57) InputAdapter +(58) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) InputIteratorTransformer +(59) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(59) SortExecTransformer +(60) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(60) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(61) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(62) Scan parquet +(63) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(63) Filter +(64) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(64) BroadcastExchange +(65) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(65) Scan parquet +(66) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(67) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(68) Project +(69) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(69) Scan parquet +(70) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(71) BroadcastExchange +(72) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(73) Project +(74) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(75) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(78) Project +(79) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(79) Scan parquet +(80) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(80) Filter +(81) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(81) BroadcastExchange +(82) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(83) Project +(84) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(84) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(85) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(86) BroadcastExchange +(87) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(88) Project +(89) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(89) HashAggregate +(90) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(90) Exchange +(91) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(92) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(92) Exchange +(93) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Sort +(94) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(94) AdaptiveSparkPlan +(95) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt index bcdebad75e77..6c2c8eb46bc9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt @@ -1,117 +1,118 @@ == Physical Plan == -AdaptiveSparkPlan (130) +AdaptiveSparkPlan (131) +- == Final Plan == - VeloxColumnarToRowExec (85) - +- ^ SortExecTransformer (83) - +- ^ InputIteratorTransformer (82) - +- ShuffleQueryStage (80), Statistics(X) - +- ColumnarExchange (79) - +- VeloxAppendBatches (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ FilterExecTransformer (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26), Statistics(X) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ FilterExecTransformer (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35), Statistics(X) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ FilterExecTransformer (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44), Statistics(X) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ FilterExecTransformer (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53), Statistics(X) - : +- ColumnarBroadcastExchange (52) - : +- ^ FilterExecTransformer (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63), Statistics(X) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ FilterExecTransformer (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (86) + +- ^ SortExecTransformer (84) + +- ^ InputIteratorTransformer (83) + +- ShuffleQueryStage (81), Statistics(X) + +- ColumnarExchange (80) + +- VeloxAppendBatches (79) + +- ^ ProjectExecTransformer (77) + +- ^ RegularHashAggregateExecTransformer (76) + +- ^ InputIteratorTransformer (75) + +- ShuffleQueryStage (73), Statistics(X) + +- ColumnarExchange (72) + +- VeloxAppendBatches (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ FilterExecTransformer (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ FilterExecTransformer (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ FilterExecTransformer (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ FilterExecTransformer (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ FilterExecTransformer (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ FilterExecTransformer (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ FilterExecTransformer (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (129) - +- Exchange (128) - +- HashAggregate (127) - +- Exchange (126) - +- HashAggregate (125) - +- Project (124) - +- BroadcastHashJoin Inner BuildRight (123) - :- Project (118) - : +- BroadcastHashJoin Inner BuildRight (117) - : :- Project (113) - : : +- BroadcastHashJoin Inner BuildRight (112) - : : :- Project (108) - : : : +- BroadcastHashJoin Inner BuildRight (107) - : : : :- Project (103) - : : : : +- BroadcastHashJoin Inner BuildRight (102) - : : : : :- Project (98) - : : : : : +- BroadcastHashJoin Inner BuildRight (97) - : : : : : :- Project (93) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (92) - : : : : : : :- BroadcastExchange (89) - : : : : : : : +- Project (88) - : : : : : : : +- Filter (87) - : : : : : : : +- Scan parquet (86) - : : : : : : +- Filter (91) - : : : : : : +- Scan parquet (90) - : : : : : +- BroadcastExchange (96) - : : : : : +- Filter (95) - : : : : : +- Scan parquet (94) - : : : : +- BroadcastExchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- BroadcastExchange (106) - : : : +- Filter (105) - : : : +- Scan parquet (104) - : : +- BroadcastExchange (111) - : : +- Filter (110) - : : +- Scan parquet (109) - : +- BroadcastExchange (116) - : +- Filter (115) - : +- Scan parquet (114) - +- BroadcastExchange (122) - +- Project (121) - +- Filter (120) - +- Scan parquet (119) + Sort (130) + +- Exchange (129) + +- HashAggregate (128) + +- Exchange (127) + +- HashAggregate (126) + +- Project (125) + +- BroadcastHashJoin Inner BuildRight (124) + :- Project (119) + : +- BroadcastHashJoin Inner BuildRight (118) + : :- Project (114) + : : +- BroadcastHashJoin Inner BuildRight (113) + : : :- Project (109) + : : : +- BroadcastHashJoin Inner BuildRight (108) + : : : :- Project (104) + : : : : +- BroadcastHashJoin Inner BuildRight (103) + : : : : :- Project (99) + : : : : : +- BroadcastHashJoin Inner BuildRight (98) + : : : : : :- Project (94) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (93) + : : : : : : :- BroadcastExchange (90) + : : : : : : : +- Project (89) + : : : : : : : +- Filter (88) + : : : : : : : +- Scan parquet (87) + : : : : : : +- Filter (92) + : : : : : : +- Scan parquet (91) + : : : : : +- BroadcastExchange (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- BroadcastExchange (102) + : : : : +- Filter (101) + : : : : +- Scan parquet (100) + : : : +- BroadcastExchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- BroadcastExchange (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- BroadcastExchange (117) + : +- Filter (116) + : +- Scan parquet (115) + +- BroadcastExchange (123) + +- Project (122) + +- Filter (121) + +- Scan parquet (120) (1) Scan parquet @@ -421,284 +422,288 @@ Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(71) ColumnarExchange +(71) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(72) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(72) ShuffleQueryStage +(73) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(73) InputAdapter +(74) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(74) InputIteratorTransformer +(75) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(75) RegularHashAggregateExecTransformer +(76) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(76) ProjectExecTransformer +(77) ProjectExecTransformer Output [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(77) WholeStageCodegenTransformer (X) +(78) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) VeloxAppendBatches +(79) VeloxAppendBatches Input [2]: [o_year#X, mkt_share#X] Arguments: X -(79) ColumnarExchange +(80) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(80) ShuffleQueryStage +(81) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(81) InputAdapter +(82) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(82) InputIteratorTransformer +(83) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(83) SortExecTransformer +(84) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(84) WholeStageCodegenTransformer (X) +(85) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(85) VeloxColumnarToRowExec +(86) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(86) Scan parquet +(87) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(87) Filter +(88) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(88) Project +(89) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(89) BroadcastExchange +(90) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(90) Scan parquet +(91) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(91) Filter +(92) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(92) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(93) Project +(94) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(94) Scan parquet +(95) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(95) Filter +(96) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(96) BroadcastExchange +(97) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(97) BroadcastHashJoin +(98) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(98) Project +(99) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(99) Scan parquet +(100) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(100) Filter +(101) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(101) BroadcastExchange +(102) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(102) BroadcastHashJoin +(103) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(103) Project +(104) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(104) Scan parquet +(105) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(105) Filter +(106) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(106) BroadcastExchange +(107) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(107) BroadcastHashJoin +(108) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(108) Project +(109) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(109) Scan parquet +(110) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(110) Filter +(111) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(111) BroadcastExchange +(112) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(112) BroadcastHashJoin +(113) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(113) Project +(114) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(114) Scan parquet +(115) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(115) Filter +(116) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(116) BroadcastExchange +(117) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(117) BroadcastHashJoin +(118) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(118) Project +(119) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(119) Scan parquet +(120) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(120) Filter +(121) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(121) Project +(122) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(122) BroadcastExchange +(123) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(123) BroadcastHashJoin +(124) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(124) Project +(125) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(125) HashAggregate +(126) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(126) Exchange +(127) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) HashAggregate +(128) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(128) Exchange +(129) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Sort +(130) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(130) AdaptiveSparkPlan +(131) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt index 4a0f66b2e746..6a4faab249c8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt @@ -1,90 +1,91 @@ == Physical Plan == -AdaptiveSparkPlan (99) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- VeloxAppendBatches (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61), Statistics(X) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (98) - +- Exchange (97) - +- HashAggregate (96) - +- Exchange (95) - +- HashAggregate (94) - +- Project (93) - +- BroadcastHashJoin Inner BuildRight (92) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (69) - : : : : : +- Project (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Filter (71) - : : : : +- Scan parquet (70) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (91) - +- Filter (90) - +- Scan parquet (89) + Sort (99) + +- Exchange (98) + +- HashAggregate (97) + +- Exchange (96) + +- HashAggregate (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (70) + : : : : : +- Project (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -312,226 +313,230 @@ Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(53) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) VeloxAppendBatches +(59) VeloxAppendBatches Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(59) ColumnarExchange +(60) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(61) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(61) InputAdapter +(62) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) InputIteratorTransformer +(63) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(63) SortExecTransformer +(64) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(65) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(67) Filter +(68) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(68) Project +(69) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(69) BroadcastExchange +(70) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(70) Scan parquet +(71) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(73) Project +(74) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(74) Scan parquet +(75) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(78) Project +(79) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(79) Scan parquet +(80) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(80) Filter +(81) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(81) BroadcastExchange +(82) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(83) Project +(84) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(84) Scan parquet +(85) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(86) BroadcastExchange +(87) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(88) Project +(89) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(89) Scan parquet +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(91) BroadcastExchange +(92) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(92) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(93) Project +(94) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(94) HashAggregate +(95) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(95) Exchange +(96) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) HashAggregate +(97) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(97) Exchange +(98) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Sort +(99) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(99) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt index 69a930ea8545..53edb933c1fb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (29) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (28) - +- Exchange (27) - +- HashAggregate (26) - +- Exchange (25) - +- HashAggregate (24) - +- Project (23) - +- Filter (22) - +- Scan parquet (21) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -57,101 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) VeloxAppendBatches +(14) VeloxAppendBatches Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) ColumnarExchange +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(18) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(20) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(21) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(22) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(23) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(24) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(25) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(27) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(29) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt index 5d0c6e67cb0d..85176d8c6011 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt @@ -1,84 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (66) - +- TakeOrderedAndProjectExecTransformer (65) - +- ^ ProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59) - +- ColumnarExchange (58) - +- ^ ProjectExecTransformer (56) - +- ^ FlushableHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - :- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41) - : +- ColumnarExchange (40) - : +- VeloxAppendBatches (39) - : +- ^ ProjectExecTransformer (37) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - : :- ^ InputIteratorTransformer (26) - : : +- ShuffleQueryStage (24) - : : +- ColumnarExchange (23) - : : +- VeloxAppendBatches (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : : :- ^ InputIteratorTransformer (9) - : : : +- ShuffleQueryStage (7) - : : : +- ColumnarExchange (6) - : : : +- VeloxAppendBatches (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (18) - : : +- ShuffleQueryStage (16) - : : +- ColumnarExchange (15) - : : +- VeloxAppendBatches (14) - : : +- ^ ProjectExecTransformer (12) - : : +- ^ NoopFilter (11) - : : +- ^ Scan parquet (10) - : +- ^ InputIteratorTransformer (35) - : +- ShuffleQueryStage (33) - : +- ColumnarExchange (32) - : +- VeloxAppendBatches (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ NoopFilter (28) - : +- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50) - +- ColumnarExchange (49) - +- VeloxAppendBatches (48) - +- ^ ProjectExecTransformer (46) - +- ^ NoopFilter (45) - +- ^ Scan parquet (44) + VeloxColumnarToRowExec (67) + +- TakeOrderedAndProjectExecTransformer (66) + +- ^ ProjectExecTransformer (64) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ NoopFilter (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ NoopFilter (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ NoopFilter (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (92) - +- HashAggregate (91) - +- Exchange (90) - +- HashAggregate (89) - +- Project (88) - +- ShuffledHashJoin Inner BuildRight (87) - :- Exchange (83) - : +- Project (82) - : +- ShuffledHashJoin Inner BuildRight (81) - : :- Exchange (76) - : : +- Project (75) - : : +- ShuffledHashJoin Inner BuildRight (74) - : : :- Exchange (69) - : : : +- Filter (68) - : : : +- Scan parquet (67) - : : +- Exchange (73) - : : +- Project (72) - : : +- Filter (71) - : : +- Scan parquet (70) - : +- Exchange (80) - : +- Project (79) - : +- Filter (78) - : +- Scan parquet (77) - +- Exchange (86) - +- Filter (85) - +- Scan parquet (84) + TakeOrderedAndProject (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- ShuffledHashJoin Inner BuildRight (88) + :- Exchange (84) + : +- Project (83) + : +- ShuffledHashJoin Inner BuildRight (82) + : :- Exchange (77) + : : +- Project (76) + : : +- ShuffledHashJoin Inner BuildRight (75) + : : :- Exchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- Exchange (74) + : : +- Project (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (81) + : +- Project (80) + : +- Filter (79) + : +- Scan parquet (78) + +- Exchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -315,167 +316,171 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(58) ColumnarExchange +(58) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(59) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(60) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(60) InputAdapter +(61) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(61) InputIteratorTransformer +(62) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(62) RegularHashAggregateExecTransformer +(63) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(63) ProjectExecTransformer +(64) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(65) TakeOrderedAndProjectExecTransformer +(66) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(66) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) Scan parquet +(68) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(68) Filter +(69) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(69) Exchange +(70) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(70) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(72) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(73) Exchange +(74) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) ShuffledHashJoin +(75) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(75) Project +(76) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(76) Exchange +(77) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Scan parquet +(78) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(78) Filter +(79) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(79) Project +(80) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(80) Exchange +(81) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(82) Project +(83) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(83) Exchange +(84) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(86) Exchange +(87) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) ShuffledHashJoin +(88) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) Project +(89) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(89) HashAggregate +(90) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(90) Exchange +(91) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(92) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(92) TakeOrderedAndProject +(93) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(93) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt index eecba8c53af6..7c749c0a5ec6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt @@ -1,70 +1,71 @@ == Physical Plan == -AdaptiveSparkPlan (77) +AdaptiveSparkPlan (78) +- == Final Plan == - VeloxColumnarToRowExec (55) - +- ^ SortExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50) - +- ColumnarExchange (49) - +- VeloxAppendBatches (48) - +- ^ FilterExecTransformer (46) - +- ^ RegularHashAggregateExecTransformer (45) - +- ^ InputIteratorTransformer (44) - +- ShuffleQueryStage (42) - +- ColumnarExchange (41) - +- ^ ProjectExecTransformer (39) - +- ^ FlushableHashAggregateExecTransformer (38) - +- ^ ProjectExecTransformer (37) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - :- ^ InputIteratorTransformer (26) - : +- ShuffleQueryStage (24) - : +- ColumnarExchange (23) - : +- VeloxAppendBatches (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : :- ^ InputIteratorTransformer (9) - : : +- ShuffleQueryStage (7) - : : +- ColumnarExchange (6) - : : +- VeloxAppendBatches (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (18) - : +- ShuffleQueryStage (16) - : +- ColumnarExchange (15) - : +- VeloxAppendBatches (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ NoopFilter (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33) - +- ColumnarExchange (32) - +- VeloxAppendBatches (31) - +- ^ ProjectExecTransformer (29) - +- ^ NoopFilter (28) - +- ^ Scan parquet (27) + VeloxColumnarToRowExec (56) + +- ^ SortExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ShuffleQueryStage (51) + +- ColumnarExchange (50) + +- VeloxAppendBatches (49) + +- ^ FilterExecTransformer (47) + +- ^ RegularHashAggregateExecTransformer (46) + +- ^ InputIteratorTransformer (45) + +- ShuffleQueryStage (43) + +- ColumnarExchange (42) + +- VeloxAppendBatches (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ NoopFilter (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (76) - +- Exchange (75) - +- Filter (74) - +- HashAggregate (73) - +- Exchange (72) - +- HashAggregate (71) - +- Project (70) - +- ShuffledHashJoin Inner BuildRight (69) - :- Exchange (64) - : +- Project (63) - : +- ShuffledHashJoin Inner BuildRight (62) - : :- Exchange (58) - : : +- Filter (57) - : : +- Scan parquet (56) - : +- Exchange (61) - : +- Filter (60) - : +- Scan parquet (59) - +- Exchange (68) - +- Project (67) - +- Filter (66) - +- Scan parquet (65) + Sort (77) + +- Exchange (76) + +- Filter (75) + +- HashAggregate (74) + +- Exchange (73) + +- HashAggregate (72) + +- Project (71) + +- ShuffledHashJoin Inner BuildRight (70) + :- Exchange (65) + : +- Project (64) + : +- ShuffledHashJoin Inner BuildRight (63) + : :- Exchange (59) + : : +- Filter (58) + : : +- Scan parquet (57) + : +- Exchange (62) + : +- Filter (61) + : +- Scan parquet (60) + +- Exchange (69) + +- Project (68) + +- Filter (67) + +- Scan parquet (66) (1) Scan parquet @@ -233,165 +234,169 @@ Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(41) ColumnarExchange +(41) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(42) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(42) ShuffleQueryStage +(43) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(43) InputAdapter +(44) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(44) InputIteratorTransformer +(45) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(45) RegularHashAggregateExecTransformer +(46) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(46) FilterExecTransformer +(47) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(47) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(48) VeloxAppendBatches +(49) VeloxAppendBatches Input [2]: [ps_partkey#X, value#X] Arguments: X -(49) ColumnarExchange +(50) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(51) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(51) InputAdapter +(52) InputAdapter Input [2]: [ps_partkey#X, value#X] -(52) InputIteratorTransformer +(53) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(53) SortExecTransformer +(54) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(54) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(55) VeloxColumnarToRowExec +(56) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(56) Scan parquet +(57) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(57) Filter +(58) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(58) Exchange +(59) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) Scan parquet +(60) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(60) Filter +(61) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(61) Exchange +(62) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) ShuffledHashJoin +(63) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(63) Project +(64) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(64) Exchange +(65) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) Scan parquet +(66) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(67) Project +(68) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(68) Exchange +(69) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(69) ShuffledHashJoin +(70) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(70) Project +(71) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(71) HashAggregate +(72) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(72) Exchange +(73) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(73) HashAggregate +(74) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(74) Filter +(75) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(75) Exchange +(76) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) Sort +(77) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(77) AdaptiveSparkPlan +(78) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt index e6f809047498..5cf27c6e0cb9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt @@ -1,49 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32) - +- ColumnarExchange (31) - +- VeloxAppendBatches (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- VeloxAppendBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- Project (46) - +- ShuffledHashJoin Inner BuildLeft (45) - :- Exchange (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Exchange (44) - +- Project (43) - +- Filter (42) - +- Scan parquet (41) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin Inner BuildLeft (46) + :- Exchange (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -144,129 +145,133 @@ Input [3]: [l_shipmode#X, sum#X, sum#X] Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(24) ColumnarExchange +(24) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(25) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(28) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(29) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(30) VeloxAppendBatches +(31) VeloxAppendBatches Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(31) ColumnarExchange +(32) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) SortExecTransformer +(36) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(37) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(38) Scan parquet +(39) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(40) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(42) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(42) Filter +(43) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(43) Project +(44) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(44) Exchange +(45) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(46) Project +(47) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(47) HashAggregate +(48) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(48) Exchange +(49) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(50) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(50) Exchange +(51) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(52) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(52) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt index b716b8f475ba..d3904d8d079e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (55) +AdaptiveSparkPlan (56) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- ^ SortExecTransformer (37) - +- ^ InputIteratorTransformer (36) - +- ShuffleQueryStage (34) - +- ColumnarExchange (33) - +- VeloxAppendBatches (32) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- VeloxAppendBatches (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (54) - +- Exchange (53) - +- HashAggregate (52) - +- Exchange (51) - +- HashAggregate (50) - +- HashAggregate (49) - +- HashAggregate (48) - +- Project (47) - +- ShuffledHashJoin LeftOuter BuildRight (46) - :- Exchange (41) - : +- Scan parquet (40) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftOuter BuildRight (47) + :- Exchange (42) + : +- Scan parquet (41) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -160,138 +161,142 @@ Input [2]: [c_count#X, count#X] Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [2]: [c_count#X, count#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(32) VeloxAppendBatches +(33) VeloxAppendBatches Input [2]: [c_count#X, custdist#X] Arguments: X -(33) ColumnarExchange +(34) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(34) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(35) InputAdapter +(36) InputAdapter Input [2]: [c_count#X, custdist#X] -(36) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(37) SortExecTransformer +(38) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(38) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(39) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(40) Scan parquet +(41) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(41) Exchange +(42) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Scan parquet +(43) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(43) Filter +(44) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(44) Project +(45) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(45) Exchange +(46) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(47) Project +(48) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(48) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(49) HashAggregate +(50) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(50) HashAggregate +(51) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(51) Exchange +(52) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) HashAggregate +(53) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(53) Exchange +(54) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Sort +(55) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(55) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt index dbb20979b70e..eab0e2908a10 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt @@ -1,14 +1,14 @@ == Physical Plan == -AdaptiveSparkPlan (47) +AdaptiveSparkPlan (48) +- == Final Plan == - VeloxColumnarToRowExec (32) - +- ^ SortExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- VeloxAppendBatches (25) - +- ^ ProjectExecTransformer (23) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (22) + VeloxColumnarToRowExec (33) + +- ^ SortExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (23) :- ^ InputIteratorTransformer (9) : +- ShuffleQueryStage (7) : +- ColumnarExchange (6) @@ -16,31 +16,32 @@ AdaptiveSparkPlan (47) : +- ^ ProjectExecTransformer (3) : +- ^ NoopFilter (2) : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (46) - +- Exchange (45) - +- Project (44) - +- ShuffledHashJoin Inner BuildLeft (43) - :- Exchange (35) - : +- Filter (34) - : +- Scan parquet (33) - +- Filter (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- Filter (37) - +- Scan parquet (36) + Sort (47) + +- Exchange (46) + +- Project (45) + +- ShuffledHashJoin Inner BuildLeft (44) + :- Exchange (36) + : +- Filter (35) + : +- Scan parquet (34) + +- Filter (43) + +- HashAggregate (42) + +- Exchange (41) + +- HashAggregate (40) + +- Project (39) + +- Filter (38) + +- Scan parquet (37) (1) Scan parquet @@ -110,142 +111,146 @@ Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(16) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(21) FilterExecTransformer +(22) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(22) ShuffledHashJoinExecTransformer +(23) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(23) ProjectExecTransformer +(24) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(24) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(25) VeloxAppendBatches +(26) VeloxAppendBatches Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) ColumnarExchange +(27) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(30) SortExecTransformer +(31) SortExecTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(32) VeloxColumnarToRowExec +(33) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(33) Scan parquet +(34) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(34) Filter +(35) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(35) Exchange +(36) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) Scan parquet +(37) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(37) Filter +(38) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(38) Project +(39) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(39) HashAggregate +(40) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(40) Exchange +(41) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) HashAggregate +(42) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(42) Filter +(43) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(43) ShuffledHashJoin +(44) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(44) Project +(45) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(45) Exchange +(46) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) Sort +(47) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(47) AdaptiveSparkPlan +(48) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt index e3570601883a..354bd4f3fabd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt @@ -1,62 +1,64 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (45) - +- ^ SortExecTransformer (43) - +- ^ InputIteratorTransformer (42) - +- ShuffleQueryStage (40) - +- ColumnarExchange (39) - +- VeloxAppendBatches (38) - +- ^ RegularHashAggregateExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33) - +- ColumnarExchange (32) - +- ^ ProjectExecTransformer (30) - +- ^ FlushableHashAggregateExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- VeloxAppendBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (47) + +- ^ SortExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42) + +- ColumnarExchange (41) + +- VeloxAppendBatches (40) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ ProjectExecTransformer (31) + +- ^ FlushableHashAggregateExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (66) - +- Exchange (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- Project (58) - +- ShuffledHashJoin Inner BuildRight (57) - :- Exchange (53) - : +- BroadcastHashJoin LeftAnti BuildRight (52) - : :- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (51) - : +- Project (50) - : +- Filter (49) - : +- Scan parquet (48) - +- Exchange (56) - +- Filter (55) - +- Scan parquet (54) + Sort (68) + +- Exchange (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- HashAggregate (63) + +- Exchange (62) + +- HashAggregate (61) + +- Project (60) + +- ShuffledHashJoin Inner BuildRight (59) + :- Exchange (55) + : +- BroadcastHashJoin LeftAnti BuildRight (54) + : :- Filter (49) + : : +- Scan parquet (48) + : +- BroadcastExchange (53) + : +- Project (52) + : +- Filter (51) + : +- Scan parquet (50) + +- Exchange (58) + +- Filter (57) + +- Scan parquet (56) (1) Scan parquet @@ -157,203 +159,211 @@ Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(24) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(28) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(29) FlushableHashAggregateExecTransformer +(30) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(30) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(32) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(34) InputAdapter +(36) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(35) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(36) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(37) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(38) VeloxAppendBatches +(40) VeloxAppendBatches Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(39) ColumnarExchange +(41) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(42) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(41) InputAdapter +(43) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(42) InputIteratorTransformer +(44) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) SortExecTransformer +(45) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(44) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(45) VeloxColumnarToRowExec +(47) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(46) Scan parquet +(48) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(47) Filter +(49) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(48) Scan parquet +(50) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(49) Filter +(51) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(50) Project +(52) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(51) BroadcastExchange +(53) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(52) BroadcastHashJoin +(54) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(53) Exchange +(55) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(56) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(56) Exchange +(58) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(59) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(58) Project +(60) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(59) HashAggregate +(61) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(60) Exchange +(62) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(63) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(62) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(63) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(66) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(65) Exchange +(67) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) Sort +(68) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(67) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt index b79bd948bec8..848d4e2ce4f8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt @@ -1,12 +1,12 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- ^ ProjectExecTransformer (37) - +- ^ RegularHashAggregateExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ ProjectExecTransformer (34) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (33) + VeloxColumnarToRowExec (40) + +- ^ ProjectExecTransformer (38) + +- ^ RegularHashAggregateExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ ProjectExecTransformer (35) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (34) :- ^ ProjectExecTransformer (20) : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) : :- ^ InputIteratorTransformer (9) @@ -23,36 +23,37 @@ AdaptiveSparkPlan (59) : +- ^ ProjectExecTransformer (12) : +- ^ NoopFilter (11) : +- ^ Scan parquet (10) - +- ^ FilterExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ NoopFilter (22) - +- ^ Scan parquet (21) + +- ^ FilterExecTransformer (33) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ NoopFilter (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (58) - +- HashAggregate (57) - +- Project (56) - +- ShuffledHashJoin Inner BuildRight (55) - :- Project (48) - : +- ShuffledHashJoin Inner BuildRight (47) - : :- Exchange (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Project (45) - : +- Filter (44) - : +- Scan parquet (43) - +- Filter (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- Filter (50) - +- Scan parquet (49) + HashAggregate (59) + +- HashAggregate (58) + +- Project (57) + +- ShuffledHashJoin Inner BuildRight (56) + :- Project (49) + : +- ShuffledHashJoin Inner BuildRight (48) + : :- Exchange (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Exchange (47) + : +- Project (46) + : +- Filter (45) + : +- Scan parquet (44) + +- Filter (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- Filter (51) + +- Scan parquet (50) (1) Scan parquet @@ -164,168 +165,172 @@ Input [3]: [l_partkey#X, sum#X, count#X] Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(32) FilterExecTransformer +(33) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(33) ShuffledHashJoinExecTransformer +(34) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(34) ProjectExecTransformer +(35) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(35) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(36) RegularHashAggregateExecTransformer +(37) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(37) ProjectExecTransformer +(38) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(38) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(39) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(40) Scan parquet +(41) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(41) Filter +(42) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(42) Exchange +(43) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Scan parquet +(44) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(45) Project +(46) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(46) Exchange +(47) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(48) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(48) Project +(49) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(49) Scan parquet +(50) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(50) Filter +(51) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(51) HashAggregate +(52) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(52) Exchange +(53) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(54) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(54) Filter +(55) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(55) ShuffledHashJoin +(56) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(56) Project +(57) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(57) HashAggregate +(58) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(58) HashAggregate +(59) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] -(59) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt index a7a179a81554..3d4743403809 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt @@ -1,18 +1,18 @@ == Physical Plan == -AdaptiveSparkPlan (102) +AdaptiveSparkPlan (103) +- == Final Plan == - VeloxColumnarToRowExec (69) - +- TakeOrderedAndProjectExecTransformer (68) - +- ^ RegularHashAggregateExecTransformer (66) - +- ^ RegularHashAggregateExecTransformer (65) - +- ^ ProjectExecTransformer (64) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (63) - :- ^ InputIteratorTransformer (45) - : +- ShuffleQueryStage (43) - : +- ColumnarExchange (42) - : +- VeloxAppendBatches (41) - : +- ^ ProjectExecTransformer (39) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (38) + VeloxColumnarToRowExec (70) + +- TakeOrderedAndProjectExecTransformer (69) + +- ^ RegularHashAggregateExecTransformer (67) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ ProjectExecTransformer (65) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (64) + :- ^ InputIteratorTransformer (46) + : +- ShuffleQueryStage (44) + : +- ColumnarExchange (43) + : +- VeloxAppendBatches (42) + : +- ^ ProjectExecTransformer (40) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (39) : :- ^ InputIteratorTransformer (9) : : +- ShuffleQueryStage (7) : : +- ColumnarExchange (6) @@ -20,12 +20,12 @@ AdaptiveSparkPlan (102) : : +- ^ ProjectExecTransformer (3) : : +- ^ NoopFilter (2) : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (37) - : +- ShuffleQueryStage (35) - : +- ColumnarExchange (34) - : +- VeloxAppendBatches (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (30) + : +- ^ InputIteratorTransformer (38) + : +- ShuffleQueryStage (36) + : +- ColumnarExchange (35) + : +- VeloxAppendBatches (34) + : +- ^ ProjectExecTransformer (32) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (31) : :- ^ InputIteratorTransformer (18) : : +- ShuffleQueryStage (16) : : +- ColumnarExchange (15) @@ -33,62 +33,63 @@ AdaptiveSparkPlan (102) : : +- ^ ProjectExecTransformer (12) : : +- ^ NoopFilter (11) : : +- ^ Scan parquet (10) - : +- ^ ProjectExecTransformer (29) - : +- ^ FilterExecTransformer (28) - : +- ^ RegularHashAggregateExecTransformer (27) - : +- ^ InputIteratorTransformer (26) - : +- ShuffleQueryStage (24) - : +- ColumnarExchange (23) - : +- ^ ProjectExecTransformer (21) - : +- ^ FlushableHashAggregateExecTransformer (20) - : +- ^ Scan parquet (19) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - :- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52) - : +- ColumnarExchange (51) - : +- VeloxAppendBatches (50) - : +- ^ ProjectExecTransformer (48) - : +- ^ NoopFilter (47) - : +- ^ Scan parquet (46) - +- ^ ProjectExecTransformer (61) - +- ^ FilterExecTransformer (60) - +- ^ RegularHashAggregateExecTransformer (59) - +- ^ InputIteratorTransformer (58) - +- ShuffleQueryStage (56) - +- ReusedExchange (55) + : +- ^ ProjectExecTransformer (30) + : +- ^ FilterExecTransformer (29) + : +- ^ RegularHashAggregateExecTransformer (28) + : +- ^ InputIteratorTransformer (27) + : +- ShuffleQueryStage (25) + : +- ColumnarExchange (24) + : +- VeloxAppendBatches (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (63) + :- ^ InputIteratorTransformer (55) + : +- ShuffleQueryStage (53) + : +- ColumnarExchange (52) + : +- VeloxAppendBatches (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ NoopFilter (48) + : +- ^ Scan parquet (47) + +- ^ ProjectExecTransformer (62) + +- ^ FilterExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57) + +- ReusedExchange (56) +- == Initial Plan == - TakeOrderedAndProject (101) - +- HashAggregate (100) - +- HashAggregate (99) - +- Project (98) - +- ShuffledHashJoin Inner BuildRight (97) - :- Exchange (86) - : +- Project (85) - : +- ShuffledHashJoin Inner BuildLeft (84) - : :- Exchange (72) - : : +- Filter (71) - : : +- Scan parquet (70) - : +- Exchange (83) - : +- ShuffledHashJoin LeftSemi BuildRight (82) - : :- Exchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- Project (81) - : +- Filter (80) - : +- HashAggregate (79) - : +- Exchange (78) - : +- HashAggregate (77) - : +- Scan parquet (76) - +- ShuffledHashJoin LeftSemi BuildRight (96) - :- Exchange (89) - : +- Filter (88) - : +- Scan parquet (87) - +- Project (95) - +- Filter (94) - +- HashAggregate (93) - +- Exchange (92) - +- HashAggregate (91) - +- Scan parquet (90) + TakeOrderedAndProject (102) + +- HashAggregate (101) + +- HashAggregate (100) + +- Project (99) + +- ShuffledHashJoin Inner BuildRight (98) + :- Exchange (87) + : +- Project (86) + : +- ShuffledHashJoin Inner BuildLeft (85) + : :- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (84) + : +- ShuffledHashJoin LeftSemi BuildRight (83) + : :- Exchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Project (82) + : +- Filter (81) + : +- HashAggregate (80) + : +- Exchange (79) + : +- HashAggregate (78) + : +- Scan parquet (77) + +- ShuffledHashJoin LeftSemi BuildRight (97) + :- Exchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- Project (96) + +- Filter (95) + +- HashAggregate (94) + +- Exchange (93) + +- HashAggregate (92) + +- Scan parquet (91) (1) Scan parquet @@ -186,364 +187,368 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(28) FilterExecTransformer +(29) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(29) ProjectExecTransformer +(30) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(30) ShuffledHashJoinExecTransformer +(31) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(33) VeloxAppendBatches +(34) VeloxAppendBatches Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(34) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(35) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(36) InputAdapter +(37) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(38) ShuffledHashJoinExecTransformer +(39) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(39) ProjectExecTransformer +(40) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(40) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(41) VeloxAppendBatches +(42) VeloxAppendBatches Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(42) ColumnarExchange +(43) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(43) ShuffleQueryStage +(44) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(44) InputAdapter +(45) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(45) InputIteratorTransformer +(46) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(46) Scan parquet +(47) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(47) NoopFilter +(48) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(48) ProjectExecTransformer +(49) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(49) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(50) VeloxAppendBatches +(51) VeloxAppendBatches Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: X -(51) ColumnarExchange +(52) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(53) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(53) InputAdapter +(54) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(54) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(55) ReusedExchange [Reuses operator id: 23] +(56) ReusedExchange [Reuses operator id: 24] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(56) ShuffleQueryStage +(57) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(57) InputAdapter +(58) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(58) InputIteratorTransformer +(59) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(59) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(60) FilterExecTransformer +(61) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(61) ProjectExecTransformer +(62) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(62) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(63) ShuffledHashJoinExecTransformer +(64) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(64) ProjectExecTransformer +(65) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(65) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(66) RegularHashAggregateExecTransformer +(67) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(67) WholeStageCodegenTransformer (X) +(68) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(68) TakeOrderedAndProjectExecTransformer +(69) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(69) VeloxColumnarToRowExec +(70) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(70) Scan parquet +(71) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(72) Exchange +(73) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(73) Scan parquet +(74) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(75) Exchange +(76) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) Scan parquet +(77) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(77) HashAggregate +(78) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(78) Exchange +(79) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) HashAggregate +(80) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(80) Filter +(81) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(81) Project +(82) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(82) ShuffledHashJoin +(83) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(83) Exchange +(84) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) ShuffledHashJoin +(85) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(85) Project +(86) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(86) Exchange +(87) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Scan parquet +(88) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(88) Filter +(89) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(89) Exchange +(90) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(91) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(91) HashAggregate +(92) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(92) Exchange +(93) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) HashAggregate +(94) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(94) Filter +(95) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(95) Project +(96) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(96) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(97) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(98) Project +(99) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(99) HashAggregate +(100) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(100) HashAggregate +(101) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(101) TakeOrderedAndProject +(102) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(102) AdaptiveSparkPlan +(103) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt index 0099ce88edc8..b8c363fce329 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt @@ -1,113 +1,114 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (129) +- == Final Plan == - VeloxColumnarToRowExec (91) - +- TakeOrderedAndProjectExecTransformer (90) - +- ^ RegularHashAggregateExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85) - +- ColumnarExchange (84) - +- ^ ProjectExecTransformer (82) - +- ^ FlushableHashAggregateExecTransformer (81) - +- ^ ProjectExecTransformer (80) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) - :- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) - : : :- ^ InputIteratorTransformer (9) - : : : +- ShuffleQueryStage (7) - : : : +- ColumnarExchange (6) - : : : +- VeloxAppendBatches (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (44) - : : +- ShuffleQueryStage (42) - : : +- ColumnarExchange (41) - : : +- VeloxAppendBatches (40) - : : +- ^ ProjectExecTransformer (38) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) - : : : :- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (26) - : : : +- ShuffleQueryStage (24) - : : : +- ColumnarExchange (23) - : : : +- VeloxAppendBatches (22) - : : : +- ^ ProjectExecTransformer (20) - : : : +- ^ Scan parquet (19) - : : +- ^ InputIteratorTransformer (36) - : : +- ShuffleQueryStage (34) - : : +- ColumnarExchange (33) - : : +- VeloxAppendBatches (32) - : : +- ^ ProjectExecTransformer (30) - : : +- ^ NoopFilter (29) - : : +- ^ Scan parquet (28) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- VeloxAppendBatches (57) - : +- ^ ProjectExecTransformer (55) - : +- ^ NoopFilter (54) - : +- ^ Scan parquet (53) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76) - +- ColumnarExchange (75) - +- VeloxAppendBatches (74) - +- ^ ProjectExecTransformer (72) - +- ^ NoopFilter (71) - +- ^ Scan parquet (70) + VeloxColumnarToRowExec (92) + +- TakeOrderedAndProjectExecTransformer (91) + +- ^ RegularHashAggregateExecTransformer (89) + +- ^ InputIteratorTransformer (88) + +- ShuffleQueryStage (86) + +- ColumnarExchange (85) + +- VeloxAppendBatches (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ NoopFilter (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ NoopFilter (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ NoopFilter (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- ShuffledHashJoin Inner BuildRight (122) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (110) - : : +- Project (109) - : : +- ShuffledHashJoin Inner BuildLeft (108) - : : :- Exchange (94) - : : : +- Filter (93) - : : : +- Scan parquet (92) - : : +- Exchange (107) - : : +- ShuffledHashJoin LeftAnti BuildRight (106) - : : :- ShuffledHashJoin LeftSemi BuildRight (101) - : : : :- Exchange (98) - : : : : +- Project (97) - : : : : +- Filter (96) - : : : : +- Scan parquet (95) - : : : +- Exchange (100) - : : : +- Scan parquet (99) - : : +- Exchange (105) - : : +- Project (104) - : : +- Filter (103) - : : +- Scan parquet (102) - : +- Exchange (114) - : +- Project (113) - : +- Filter (112) - : +- Scan parquet (111) - +- Exchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + TakeOrderedAndProject (128) + +- HashAggregate (127) + +- Exchange (126) + +- HashAggregate (125) + +- Project (124) + +- ShuffledHashJoin Inner BuildRight (123) + :- Exchange (118) + : +- Project (117) + : +- ShuffledHashJoin Inner BuildRight (116) + : :- Exchange (111) + : : +- Project (110) + : : +- ShuffledHashJoin Inner BuildLeft (109) + : : :- Exchange (95) + : : : +- Filter (94) + : : : +- Scan parquet (93) + : : +- Exchange (108) + : : +- ShuffledHashJoin LeftAnti BuildRight (107) + : : :- ShuffledHashJoin LeftSemi BuildRight (102) + : : : :- Exchange (99) + : : : : +- Project (98) + : : : : +- Filter (97) + : : : : +- Scan parquet (96) + : : : +- Exchange (101) + : : : +- Scan parquet (100) + : : +- Exchange (106) + : : +- Project (105) + : : +- Filter (104) + : : +- Scan parquet (103) + : +- Exchange (115) + : +- Project (114) + : +- Filter (113) + : +- Scan parquet (112) + +- Exchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) (1) Scan parquet @@ -449,210 +450,214 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(84) ColumnarExchange +(84) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(85) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(86) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(86) InputAdapter +(87) InputAdapter Input [2]: [s_name#X, count#X] -(87) InputIteratorTransformer +(88) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(88) RegularHashAggregateExecTransformer +(89) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(89) WholeStageCodegenTransformer (X) +(90) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(90) TakeOrderedAndProjectExecTransformer +(91) TakeOrderedAndProjectExecTransformer Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X], 0 -(91) VeloxColumnarToRowExec +(92) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(92) Scan parquet +(93) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(93) Filter +(94) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(94) Exchange +(95) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Scan parquet +(96) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(96) Filter +(97) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(97) Project +(98) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(98) Exchange +(99) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(100) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(100) Exchange +(101) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) ShuffledHashJoin +(102) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(102) Scan parquet +(103) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(103) Filter +(104) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(104) Project +(105) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(105) Exchange +(106) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(107) Exchange +(108) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(109) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(109) Project +(110) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(110) Exchange +(111) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(112) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(112) Filter +(113) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(113) Project +(114) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(114) Exchange +(115) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(116) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(116) Project +(117) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(117) Exchange +(118) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(119) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(119) Filter +(120) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(120) Project +(121) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(121) Exchange +(122) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) ShuffledHashJoin +(123) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(123) Project +(124) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(124) HashAggregate +(125) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(125) Exchange +(126) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(127) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(127) TakeOrderedAndProject +(128) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(128) AdaptiveSparkPlan +(129) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt index ac5dfa762fa1..f336a73676ea 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt @@ -1,46 +1,47 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (50) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- VeloxAppendBatches (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ ProjectExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin LeftAnti BuildRight (42) - :- Exchange (39) - : +- Filter (38) - : +- Scan parquet (37) - +- Exchange (41) - +- Scan parquet (40) + Sort (49) + +- Exchange (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftAnti BuildRight (43) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (42) + +- Scan parquet (41) (1) Scan parquet @@ -136,120 +137,124 @@ Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(29) VeloxAppendBatches +(30) VeloxAppendBatches Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(30) ColumnarExchange +(31) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(32) InputAdapter +(33) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(33) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) SortExecTransformer +(35) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(35) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(36) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(37) Scan parquet +(38) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(38) Filter +(39) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(39) Exchange +(40) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) Scan parquet +(41) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(41) Exchange +(42) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(44) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(45) Exchange +(46) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(47) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(47) Exchange +(48) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(49) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(50) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt index 0a7ae2856c03..42a8fef3563f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt @@ -1,50 +1,51 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32) - +- ColumnarExchange (31) - +- VeloxAppendBatches (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- VeloxAppendBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (52) - +- Exchange (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- ShuffledHashJoin LeftSemi BuildRight (46) - :- Exchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (53) + +- Exchange (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftSemi BuildRight (47) + :- Exchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -145,133 +146,137 @@ Input [2]: [o_orderpriority#X, count#X] Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(24) ColumnarExchange +(24) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(28) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(29) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(30) VeloxAppendBatches +(31) VeloxAppendBatches Input [2]: [o_orderpriority#X, order_count#X] Arguments: X -(31) ColumnarExchange +(32) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(35) SortExecTransformer +(36) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(37) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(41) Exchange +(42) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Scan parquet +(43) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(43) Filter +(44) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(44) Project +(45) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(45) Exchange +(46) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(47) Project +(48) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(48) HashAggregate +(49) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(49) Exchange +(50) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(51) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(51) Exchange +(52) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(53) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt index 8b26063b1527..378085655899 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt @@ -1,126 +1,127 @@ == Physical Plan == -AdaptiveSparkPlan (145) +AdaptiveSparkPlan (146) +- == Final Plan == - VeloxColumnarToRowExec (105) - +- ^ SortExecTransformer (103) - +- ^ InputIteratorTransformer (102) - +- ShuffleQueryStage (100) - +- ColumnarExchange (99) - +- VeloxAppendBatches (98) - +- ^ RegularHashAggregateExecTransformer (96) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93) - +- ColumnarExchange (92) - +- ^ ProjectExecTransformer (90) - +- ^ FlushableHashAggregateExecTransformer (89) - +- ^ ProjectExecTransformer (88) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - :- ^ InputIteratorTransformer (77) - : +- ShuffleQueryStage (75) - : +- ColumnarExchange (74) - : +- VeloxAppendBatches (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - : :- ^ InputIteratorTransformer (60) - : : +- ShuffleQueryStage (58) - : : +- ColumnarExchange (57) - : : +- VeloxAppendBatches (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - : : :- ^ InputIteratorTransformer (43) - : : : +- ShuffleQueryStage (41) - : : : +- ColumnarExchange (40) - : : : +- VeloxAppendBatches (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - : : : :- ^ InputIteratorTransformer (26) - : : : : +- ShuffleQueryStage (24) - : : : : +- ColumnarExchange (23) - : : : : +- VeloxAppendBatches (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ InputIteratorTransformer (9) - : : : : : +- ShuffleQueryStage (7) - : : : : : +- ColumnarExchange (6) - : : : : : +- VeloxAppendBatches (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (35) - : : : +- ShuffleQueryStage (33) - : : : +- ColumnarExchange (32) - : : : +- VeloxAppendBatches (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ NoopFilter (28) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ NoopFilter (45) - : : +- ^ Scan parquet (44) - : +- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ NoopFilter (62) - : +- ^ Scan parquet (61) - +- ^ InputIteratorTransformer (86) - +- ShuffleQueryStage (84) - +- ColumnarExchange (83) - +- VeloxAppendBatches (82) - +- ^ ProjectExecTransformer (80) - +- ^ NoopFilter (79) - +- ^ Scan parquet (78) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (144) - +- Exchange (143) - +- HashAggregate (142) - +- Exchange (141) - +- HashAggregate (140) - +- Project (139) - +- ShuffledHashJoin Inner BuildRight (138) - :- Exchange (133) - : +- Project (132) - : +- ShuffledHashJoin Inner BuildRight (131) - : :- Exchange (127) - : : +- Project (126) - : : +- ShuffledHashJoin Inner BuildRight (125) - : : :- Exchange (121) - : : : +- Project (120) - : : : +- ShuffledHashJoin Inner BuildRight (119) - : : : :- Exchange (115) - : : : : +- Project (114) - : : : : +- ShuffledHashJoin Inner BuildLeft (113) - : : : : :- Exchange (108) - : : : : : +- Filter (107) - : : : : : +- Scan parquet (106) - : : : : +- Exchange (112) - : : : : +- Project (111) - : : : : +- Filter (110) - : : : : +- Scan parquet (109) - : : : +- Exchange (118) - : : : +- Filter (117) - : : : +- Scan parquet (116) - : : +- Exchange (124) - : : +- Filter (123) - : : +- Scan parquet (122) - : +- Exchange (130) - : +- Filter (129) - : +- Scan parquet (128) - +- Exchange (137) - +- Project (136) - +- Filter (135) - +- Scan parquet (134) + Sort (145) + +- Exchange (144) + +- HashAggregate (143) + +- Exchange (142) + +- HashAggregate (141) + +- Project (140) + +- ShuffledHashJoin Inner BuildRight (139) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Project (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (138) + +- Project (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -493,245 +494,249 @@ Input [3]: [n_name#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(92) ColumnarExchange +(92) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(93) ShuffleQueryStage +(94) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(94) InputAdapter +(95) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(95) InputIteratorTransformer +(96) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(96) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(97) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(98) VeloxAppendBatches +(99) VeloxAppendBatches Input [2]: [n_name#X, revenue#X] Arguments: X -(99) ColumnarExchange +(100) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(100) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(101) InputAdapter +(102) InputAdapter Input [2]: [n_name#X, revenue#X] -(102) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(103) SortExecTransformer +(104) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(104) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(105) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(106) Scan parquet +(107) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(108) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(109) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Scan parquet +(110) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(110) Filter +(111) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(111) Project +(112) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(112) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(114) Project +(115) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(115) Exchange +(116) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) Scan parquet +(117) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(117) Filter +(118) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(118) Exchange +(119) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(120) Project +(121) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(121) Exchange +(122) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(123) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(123) Filter +(124) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(124) Exchange +(125) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(126) Project +(127) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(127) Exchange +(128) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(129) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(129) Filter +(130) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(130) Exchange +(131) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(132) Project +(133) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(133) Exchange +(134) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(134) Scan parquet +(135) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(135) Filter +(136) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(136) Project +(137) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(137) Exchange +(138) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(138) ShuffledHashJoin +(139) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(139) Project +(140) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(140) HashAggregate +(141) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(141) Exchange +(142) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) HashAggregate +(143) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(143) Exchange +(144) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(144) Sort +(145) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(145) AdaptiveSparkPlan +(146) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt index a5022343944f..b6f876d48e5a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt @@ -7,7 +7,7 @@ AdaptiveSparkPlan (20) +- ShuffleQueryStage (8) +- ColumnarExchange (7) +- VeloxAppendBatches (6) - +- ^ RegularHashAggregateExecTransformer (4) + +- ^ FlushableHashAggregateExecTransformer (4) +- ^ ProjectExecTransformer (3) +- ^ NoopFilter (2) +- ^ Scan parquet (1) @@ -35,7 +35,7 @@ Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) RegularHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt index a7a157760c7c..a7054770a17e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt @@ -1,120 +1,121 @@ == Physical Plan == -AdaptiveSparkPlan (138) +AdaptiveSparkPlan (139) +- == Final Plan == - VeloxColumnarToRowExec (100) - +- ^ SortExecTransformer (98) - +- ^ InputIteratorTransformer (97) - +- ShuffleQueryStage (95) - +- ColumnarExchange (94) - +- VeloxAppendBatches (93) - +- ^ RegularHashAggregateExecTransformer (91) - +- ^ InputIteratorTransformer (90) - +- ShuffleQueryStage (88) - +- ColumnarExchange (87) - +- ^ ProjectExecTransformer (85) - +- ^ FlushableHashAggregateExecTransformer (84) - +- ^ ProjectExecTransformer (83) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) - :- ^ InputIteratorTransformer (77) - : +- ShuffleQueryStage (75) - : +- ColumnarExchange (74) - : +- VeloxAppendBatches (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - : :- ^ InputIteratorTransformer (60) - : : +- ShuffleQueryStage (58) - : : +- ColumnarExchange (57) - : : +- VeloxAppendBatches (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - : : :- ^ InputIteratorTransformer (43) - : : : +- ShuffleQueryStage (41) - : : : +- ColumnarExchange (40) - : : : +- VeloxAppendBatches (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : :- ^ InputIteratorTransformer (26) - : : : : +- ShuffleQueryStage (24) - : : : : +- ColumnarExchange (23) - : : : : +- VeloxAppendBatches (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ InputIteratorTransformer (9) - : : : : : +- ShuffleQueryStage (7) - : : : : : +- ColumnarExchange (6) - : : : : : +- VeloxAppendBatches (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (35) - : : : +- ShuffleQueryStage (33) - : : : +- ColumnarExchange (32) - : : : +- VeloxAppendBatches (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ NoopFilter (28) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ NoopFilter (45) - : : +- ^ Scan parquet (44) - : +- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ NoopFilter (62) - : +- ^ Scan parquet (61) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79) - +- ReusedExchange (78) + VeloxColumnarToRowExec (101) + +- ^ SortExecTransformer (99) + +- ^ InputIteratorTransformer (98) + +- ShuffleQueryStage (96) + +- ColumnarExchange (95) + +- VeloxAppendBatches (94) + +- ^ RegularHashAggregateExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ShuffleQueryStage (89) + +- ColumnarExchange (88) + +- VeloxAppendBatches (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79) + +- ReusedExchange (78) +- == Initial Plan == - Sort (137) - +- Exchange (136) - +- HashAggregate (135) - +- Exchange (134) - +- HashAggregate (133) - +- Project (132) - +- ShuffledHashJoin Inner BuildRight (131) - :- Exchange (127) - : +- Project (126) - : +- ShuffledHashJoin Inner BuildRight (125) - : :- Exchange (121) - : : +- Project (120) - : : +- ShuffledHashJoin Inner BuildRight (119) - : : :- Exchange (115) - : : : +- Project (114) - : : : +- ShuffledHashJoin Inner BuildRight (113) - : : : :- Exchange (109) - : : : : +- Project (108) - : : : : +- ShuffledHashJoin Inner BuildLeft (107) - : : : : :- Exchange (103) - : : : : : +- Filter (102) - : : : : : +- Scan parquet (101) - : : : : +- Exchange (106) - : : : : +- Filter (105) - : : : : +- Scan parquet (104) - : : : +- Exchange (112) - : : : +- Filter (111) - : : : +- Scan parquet (110) - : : +- Exchange (118) - : : +- Filter (117) - : : +- Scan parquet (116) - : +- Exchange (124) - : +- Filter (123) - : +- Scan parquet (122) - +- Exchange (130) - +- Filter (129) - +- Scan parquet (128) + Sort (138) + +- Exchange (137) + +- HashAggregate (136) + +- Exchange (135) + +- HashAggregate (134) + +- Project (133) + +- ShuffledHashJoin Inner BuildRight (132) + :- Exchange (128) + : +- Project (127) + : +- ShuffledHashJoin Inner BuildRight (126) + : :- Exchange (122) + : : +- Project (121) + : : +- ShuffledHashJoin Inner BuildRight (120) + : : :- Exchange (116) + : : : +- Project (115) + : : : +- ShuffledHashJoin Inner BuildRight (114) + : : : :- Exchange (110) + : : : : +- Project (109) + : : : : +- ShuffledHashJoin Inner BuildLeft (108) + : : : : :- Exchange (104) + : : : : : +- Filter (103) + : : : : : +- Scan parquet (102) + : : : : +- Exchange (107) + : : : : +- Filter (106) + : : : : +- Scan parquet (105) + : : : +- Exchange (113) + : : : +- Filter (112) + : : : +- Scan parquet (111) + : : +- Exchange (119) + : : +- Filter (118) + : : +- Scan parquet (117) + : +- Exchange (125) + : +- Filter (124) + : +- Scan parquet (123) + +- Exchange (131) + +- Filter (130) + +- Scan parquet (129) (1) Scan parquet @@ -463,237 +464,241 @@ Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(87) ColumnarExchange +(87) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(88) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(88) ShuffleQueryStage +(89) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(89) InputAdapter +(90) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(90) InputIteratorTransformer +(91) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(91) RegularHashAggregateExecTransformer +(92) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(92) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(93) VeloxAppendBatches +(94) VeloxAppendBatches Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(94) ColumnarExchange +(95) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(95) ShuffleQueryStage +(96) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(96) InputAdapter +(97) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(97) InputIteratorTransformer +(98) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(98) SortExecTransformer +(99) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(99) WholeStageCodegenTransformer (X) +(100) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(100) VeloxColumnarToRowExec +(101) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(101) Scan parquet +(102) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(102) Filter +(103) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(103) Exchange +(104) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) Scan parquet +(105) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(105) Filter +(106) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(106) Exchange +(107) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(108) Project +(109) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(109) Exchange +(110) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(112) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(114) Project +(115) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(115) Exchange +(116) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) Scan parquet +(117) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(117) Filter +(118) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(118) Exchange +(119) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(120) Project +(121) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(121) Exchange +(122) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(123) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(123) Filter +(124) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(124) Exchange +(125) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(126) Project +(127) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(127) Exchange +(128) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(129) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(129) Filter +(130) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(130) Exchange +(131) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(132) Project +(133) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(133) HashAggregate +(134) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(134) Exchange +(135) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(135) HashAggregate +(136) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(136) Exchange +(137) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) Sort +(138) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(138) AdaptiveSparkPlan +(139) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt index 240315c7973c..cdede8445908 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt @@ -1,165 +1,166 @@ == Physical Plan == -AdaptiveSparkPlan (192) +AdaptiveSparkPlan (193) +- == Final Plan == - VeloxColumnarToRowExec (140) - +- ^ SortExecTransformer (138) - +- ^ InputIteratorTransformer (137) - +- ShuffleQueryStage (135) - +- ColumnarExchange (134) - +- VeloxAppendBatches (133) - +- ^ ProjectExecTransformer (131) - +- ^ RegularHashAggregateExecTransformer (130) - +- ^ InputIteratorTransformer (129) - +- ShuffleQueryStage (127) - +- ColumnarExchange (126) - +- ^ ProjectExecTransformer (124) - +- ^ FlushableHashAggregateExecTransformer (123) - +- ^ ProjectExecTransformer (122) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) - :- ^ InputIteratorTransformer (111) - : +- ShuffleQueryStage (109) - : +- ColumnarExchange (108) - : +- VeloxAppendBatches (107) - : +- ^ ProjectExecTransformer (105) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) - : :- ^ InputIteratorTransformer (94) - : : +- ShuffleQueryStage (92) - : : +- ColumnarExchange (91) - : : +- VeloxAppendBatches (90) - : : +- ^ ProjectExecTransformer (88) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - : : :- ^ InputIteratorTransformer (77) - : : : +- ShuffleQueryStage (75) - : : : +- ColumnarExchange (74) - : : : +- VeloxAppendBatches (73) - : : : +- ^ ProjectExecTransformer (71) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) - : : : :- ^ InputIteratorTransformer (60) - : : : : +- ShuffleQueryStage (58) - : : : : +- ColumnarExchange (57) - : : : : +- VeloxAppendBatches (56) - : : : : +- ^ ProjectExecTransformer (54) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) - : : : : :- ^ InputIteratorTransformer (43) - : : : : : +- ShuffleQueryStage (41) - : : : : : +- ColumnarExchange (40) - : : : : : +- VeloxAppendBatches (39) - : : : : : +- ^ ProjectExecTransformer (37) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : : : :- ^ InputIteratorTransformer (26) - : : : : : : +- ShuffleQueryStage (24) - : : : : : : +- ColumnarExchange (23) - : : : : : : +- VeloxAppendBatches (22) - : : : : : : +- ^ ProjectExecTransformer (20) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : : : :- ^ InputIteratorTransformer (9) - : : : : : : : +- ShuffleQueryStage (7) - : : : : : : : +- ColumnarExchange (6) - : : : : : : : +- VeloxAppendBatches (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (18) - : : : : : : +- ShuffleQueryStage (16) - : : : : : : +- ColumnarExchange (15) - : : : : : : +- VeloxAppendBatches (14) - : : : : : : +- ^ ProjectExecTransformer (12) - : : : : : : +- ^ NoopFilter (11) - : : : : : : +- ^ Scan parquet (10) - : : : : : +- ^ InputIteratorTransformer (35) - : : : : : +- ShuffleQueryStage (33) - : : : : : +- ColumnarExchange (32) - : : : : : +- VeloxAppendBatches (31) - : : : : : +- ^ ProjectExecTransformer (29) - : : : : : +- ^ NoopFilter (28) - : : : : : +- ^ Scan parquet (27) - : : : : +- ^ InputIteratorTransformer (52) - : : : : +- ShuffleQueryStage (50) - : : : : +- ColumnarExchange (49) - : : : : +- VeloxAppendBatches (48) - : : : : +- ^ ProjectExecTransformer (46) - : : : : +- ^ NoopFilter (45) - : : : : +- ^ Scan parquet (44) - : : : +- ^ InputIteratorTransformer (69) - : : : +- ShuffleQueryStage (67) - : : : +- ColumnarExchange (66) - : : : +- VeloxAppendBatches (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ NoopFilter (62) - : : : +- ^ Scan parquet (61) - : : +- ^ InputIteratorTransformer (86) - : : +- ShuffleQueryStage (84) - : : +- ColumnarExchange (83) - : : +- VeloxAppendBatches (82) - : : +- ^ ProjectExecTransformer (80) - : : +- ^ NoopFilter (79) - : : +- ^ Scan parquet (78) - : +- ^ InputIteratorTransformer (103) - : +- ShuffleQueryStage (101) - : +- ColumnarExchange (100) - : +- VeloxAppendBatches (99) - : +- ^ ProjectExecTransformer (97) - : +- ^ NoopFilter (96) - : +- ^ Scan parquet (95) - +- ^ InputIteratorTransformer (120) - +- ShuffleQueryStage (118) - +- ColumnarExchange (117) - +- VeloxAppendBatches (116) - +- ^ ProjectExecTransformer (114) - +- ^ NoopFilter (113) - +- ^ Scan parquet (112) + VeloxColumnarToRowExec (141) + +- ^ SortExecTransformer (139) + +- ^ InputIteratorTransformer (138) + +- ShuffleQueryStage (136) + +- ColumnarExchange (135) + +- VeloxAppendBatches (134) + +- ^ ProjectExecTransformer (132) + +- ^ RegularHashAggregateExecTransformer (131) + +- ^ InputIteratorTransformer (130) + +- ShuffleQueryStage (128) + +- ColumnarExchange (127) + +- VeloxAppendBatches (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ NoopFilter (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ NoopFilter (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ NoopFilter (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ NoopFilter (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ NoopFilter (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ NoopFilter (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ NoopFilter (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (191) - +- Exchange (190) - +- HashAggregate (189) - +- Exchange (188) - +- HashAggregate (187) - +- Project (186) - +- ShuffledHashJoin Inner BuildRight (185) - :- Exchange (180) - : +- Project (179) - : +- ShuffledHashJoin Inner BuildRight (178) - : :- Exchange (174) - : : +- Project (173) - : : +- ShuffledHashJoin Inner BuildRight (172) - : : :- Exchange (168) - : : : +- Project (167) - : : : +- ShuffledHashJoin Inner BuildRight (166) - : : : :- Exchange (162) - : : : : +- Project (161) - : : : : +- ShuffledHashJoin Inner BuildRight (160) - : : : : :- Exchange (156) - : : : : : +- Project (155) - : : : : : +- ShuffledHashJoin Inner BuildRight (154) - : : : : : :- Exchange (150) - : : : : : : +- Project (149) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (148) - : : : : : : :- Exchange (144) - : : : : : : : +- Project (143) - : : : : : : : +- Filter (142) - : : : : : : : +- Scan parquet (141) - : : : : : : +- Exchange (147) - : : : : : : +- Filter (146) - : : : : : : +- Scan parquet (145) - : : : : : +- Exchange (153) - : : : : : +- Filter (152) - : : : : : +- Scan parquet (151) - : : : : +- Exchange (159) - : : : : +- Filter (158) - : : : : +- Scan parquet (157) - : : : +- Exchange (165) - : : : +- Filter (164) - : : : +- Scan parquet (163) - : : +- Exchange (171) - : : +- Filter (170) - : : +- Scan parquet (169) - : +- Exchange (177) - : +- Filter (176) - : +- Scan parquet (175) - +- Exchange (184) - +- Project (183) - +- Filter (182) - +- Scan parquet (181) + Sort (192) + +- Exchange (191) + +- HashAggregate (190) + +- Exchange (189) + +- HashAggregate (188) + +- Project (187) + +- ShuffledHashJoin Inner BuildRight (186) + :- Exchange (181) + : +- Project (180) + : +- ShuffledHashJoin Inner BuildRight (179) + : :- Exchange (175) + : : +- Project (174) + : : +- ShuffledHashJoin Inner BuildRight (173) + : : :- Exchange (169) + : : : +- Project (168) + : : : +- ShuffledHashJoin Inner BuildRight (167) + : : : :- Exchange (163) + : : : : +- Project (162) + : : : : +- ShuffledHashJoin Inner BuildRight (161) + : : : : :- Exchange (157) + : : : : : +- Project (156) + : : : : : +- ShuffledHashJoin Inner BuildRight (155) + : : : : : :- Exchange (151) + : : : : : : +- Project (150) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (149) + : : : : : : :- Exchange (145) + : : : : : : : +- Project (144) + : : : : : : : +- Filter (143) + : : : : : : : +- Scan parquet (142) + : : : : : : +- Exchange (148) + : : : : : : +- Filter (147) + : : : : : : +- Scan parquet (146) + : : : : : +- Exchange (154) + : : : : : +- Filter (153) + : : : : : +- Scan parquet (152) + : : : : +- Exchange (160) + : : : : +- Filter (159) + : : : : +- Scan parquet (158) + : : : +- Exchange (166) + : : : +- Filter (165) + : : : +- Scan parquet (164) + : : +- Exchange (172) + : : +- Filter (171) + : : +- Scan parquet (170) + : +- Exchange (178) + : +- Filter (177) + : +- Scan parquet (176) + +- Exchange (185) + +- Project (184) + +- Filter (183) + +- Scan parquet (182) (1) Scan parquet @@ -668,305 +669,309 @@ Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(126) ColumnarExchange +(126) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(127) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(127) ShuffleQueryStage +(128) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(128) InputAdapter +(129) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(129) InputIteratorTransformer +(130) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(130) RegularHashAggregateExecTransformer +(131) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(131) ProjectExecTransformer +(132) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(132) WholeStageCodegenTransformer (X) +(133) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(133) VeloxAppendBatches +(134) VeloxAppendBatches Input [2]: [o_year#X, mkt_share#X] Arguments: X -(134) ColumnarExchange +(135) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(135) ShuffleQueryStage +(136) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(136) InputAdapter +(137) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(137) InputIteratorTransformer +(138) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(138) SortExecTransformer +(139) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(139) WholeStageCodegenTransformer (X) +(140) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(140) VeloxColumnarToRowExec +(141) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(141) Scan parquet +(142) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(142) Filter +(143) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(143) Project +(144) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(144) Exchange +(145) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) Scan parquet +(146) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(146) Filter +(147) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(147) Exchange +(148) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) ShuffledHashJoin +(149) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(149) Project +(150) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(150) Exchange +(151) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) Scan parquet +(152) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(152) Filter +(153) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(153) Exchange +(154) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) ShuffledHashJoin +(155) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(155) Project +(156) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(156) Exchange +(157) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) Scan parquet +(158) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(158) Filter +(159) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(159) Exchange +(160) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) ShuffledHashJoin +(161) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(161) Project +(162) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(162) Exchange +(163) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) Scan parquet +(164) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(164) Filter +(165) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(165) Exchange +(166) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) ShuffledHashJoin +(167) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(167) Project +(168) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(168) Exchange +(169) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(169) Scan parquet +(170) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(170) Filter +(171) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(171) Exchange +(172) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(172) ShuffledHashJoin +(173) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(173) Project +(174) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(174) Exchange +(175) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(175) Scan parquet +(176) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(176) Filter +(177) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(177) Exchange +(178) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(178) ShuffledHashJoin +(179) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(179) Project +(180) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(180) Exchange +(181) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(181) Scan parquet +(182) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(182) Filter +(183) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(183) Project +(184) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(184) Exchange +(185) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(185) ShuffledHashJoin +(186) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(186) Project +(187) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(187) HashAggregate +(188) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(188) Exchange +(189) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(189) HashAggregate +(190) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(190) Exchange +(191) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(191) Sort +(192) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(192) AdaptiveSparkPlan +(193) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt index 2bbb889c2ae3..11a02d0a54d2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt @@ -1,125 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (144) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (105) - +- ^ SortExecTransformer (103) - +- ^ InputIteratorTransformer (102) - +- ShuffleQueryStage (100) - +- ColumnarExchange (99) - +- VeloxAppendBatches (98) - +- ^ RegularHashAggregateExecTransformer (96) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93) - +- ColumnarExchange (92) - +- ^ ProjectExecTransformer (90) - +- ^ FlushableHashAggregateExecTransformer (89) - +- ^ ProjectExecTransformer (88) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - :- ^ InputIteratorTransformer (77) - : +- ShuffleQueryStage (75) - : +- ColumnarExchange (74) - : +- VeloxAppendBatches (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) - : :- ^ InputIteratorTransformer (60) - : : +- ShuffleQueryStage (58) - : : +- ColumnarExchange (57) - : : +- VeloxAppendBatches (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) - : : :- ^ InputIteratorTransformer (43) - : : : +- ShuffleQueryStage (41) - : : : +- ColumnarExchange (40) - : : : +- VeloxAppendBatches (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : :- ^ InputIteratorTransformer (26) - : : : : +- ShuffleQueryStage (24) - : : : : +- ColumnarExchange (23) - : : : : +- VeloxAppendBatches (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ InputIteratorTransformer (9) - : : : : : +- ShuffleQueryStage (7) - : : : : : +- ColumnarExchange (6) - : : : : : +- VeloxAppendBatches (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (35) - : : : +- ShuffleQueryStage (33) - : : : +- ColumnarExchange (32) - : : : +- VeloxAppendBatches (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ NoopFilter (28) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ NoopFilter (45) - : : +- ^ Scan parquet (44) - : +- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ NoopFilter (62) - : +- ^ Scan parquet (61) - +- ^ InputIteratorTransformer (86) - +- ShuffleQueryStage (84) - +- ColumnarExchange (83) - +- VeloxAppendBatches (82) - +- ^ ProjectExecTransformer (80) - +- ^ NoopFilter (79) - +- ^ Scan parquet (78) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (143) - +- Exchange (142) - +- HashAggregate (141) - +- Exchange (140) - +- HashAggregate (139) - +- Project (138) - +- ShuffledHashJoin Inner BuildRight (137) - :- Exchange (133) - : +- Project (132) - : +- ShuffledHashJoin Inner BuildRight (131) - : :- Exchange (127) - : : +- Project (126) - : : +- ShuffledHashJoin Inner BuildRight (125) - : : :- Exchange (121) - : : : +- Project (120) - : : : +- ShuffledHashJoin Inner BuildRight (119) - : : : :- Exchange (115) - : : : : +- Project (114) - : : : : +- ShuffledHashJoin Inner BuildLeft (113) - : : : : :- Exchange (109) - : : : : : +- Project (108) - : : : : : +- Filter (107) - : : : : : +- Scan parquet (106) - : : : : +- Exchange (112) - : : : : +- Filter (111) - : : : : +- Scan parquet (110) - : : : +- Exchange (118) - : : : +- Filter (117) - : : : +- Scan parquet (116) - : : +- Exchange (124) - : : +- Filter (123) - : : +- Scan parquet (122) - : +- Exchange (130) - : +- Filter (129) - : +- Scan parquet (128) - +- Exchange (136) - +- Filter (135) - +- Scan parquet (134) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (110) + : : : : : +- Project (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Filter (112) + : : : : +- Scan parquet (111) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -492,241 +493,245 @@ Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(92) ColumnarExchange +(92) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(93) ShuffleQueryStage +(94) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(94) InputAdapter +(95) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(95) InputIteratorTransformer +(96) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(96) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(97) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(98) VeloxAppendBatches +(99) VeloxAppendBatches Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(99) ColumnarExchange +(100) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(100) ShuffleQueryStage +(101) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(101) InputAdapter +(102) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(102) InputIteratorTransformer +(103) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(103) SortExecTransformer +(104) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(104) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(105) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(106) Scan parquet +(107) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(107) Filter +(108) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(108) Project +(109) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(109) Exchange +(110) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(111) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(112) Exchange +(113) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(114) Project +(115) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(115) Exchange +(116) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) Scan parquet +(117) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(117) Filter +(118) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(118) Exchange +(119) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(120) Project +(121) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(121) Exchange +(122) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(123) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(123) Filter +(124) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(124) Exchange +(125) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(126) Project +(127) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(127) Exchange +(128) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(129) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(129) Filter +(130) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(130) Exchange +(131) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(132) Project +(133) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(133) Exchange +(134) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(134) Scan parquet +(135) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(135) Filter +(136) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(136) Exchange +(137) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(138) Project +(139) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(139) HashAggregate +(140) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(140) Exchange +(141) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(141) HashAggregate +(142) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(142) Exchange +(143) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(143) Sort +(144) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(144) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt index 5ad7bc9d3675..afea15af53d2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (29) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (28) - +- Exchange (27) - +- HashAggregate (26) - +- Exchange (25) - +- HashAggregate (24) - +- Project (23) - +- Filter (22) - +- Scan parquet (21) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -57,101 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) VeloxAppendBatches +(14) VeloxAppendBatches Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) ColumnarExchange +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(18) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(20) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(21) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(22) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(23) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(24) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(25) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(27) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(29) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt index f83e7572840a..5a74265ab590 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt @@ -1,84 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (66) - +- TakeOrderedAndProjectExecTransformer (65) - +- ^ ProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ ProjectExecTransformer (56) - +- ^ FlushableHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - :- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41), Statistics(X) - : +- ColumnarExchange (40) - : +- VeloxAppendBatches (39) - : +- ^ ProjectExecTransformer (37) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - : :- ^ InputIteratorTransformer (26) - : : +- ShuffleQueryStage (24), Statistics(X) - : : +- ColumnarExchange (23) - : : +- VeloxAppendBatches (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : : :- ^ InputIteratorTransformer (9) - : : : +- ShuffleQueryStage (7), Statistics(X) - : : : +- ColumnarExchange (6) - : : : +- VeloxAppendBatches (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (18) - : : +- ShuffleQueryStage (16), Statistics(X) - : : +- ColumnarExchange (15) - : : +- VeloxAppendBatches (14) - : : +- ^ ProjectExecTransformer (12) - : : +- ^ NoopFilter (11) - : : +- ^ Scan parquet (10) - : +- ^ InputIteratorTransformer (35) - : +- ShuffleQueryStage (33), Statistics(X) - : +- ColumnarExchange (32) - : +- VeloxAppendBatches (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ NoopFilter (28) - : +- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- VeloxAppendBatches (48) - +- ^ ProjectExecTransformer (46) - +- ^ NoopFilter (45) - +- ^ Scan parquet (44) + VeloxColumnarToRowExec (67) + +- TakeOrderedAndProjectExecTransformer (66) + +- ^ ProjectExecTransformer (64) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24), Statistics(X) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ NoopFilter (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33), Statistics(X) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ NoopFilter (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ NoopFilter (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (92) - +- HashAggregate (91) - +- Exchange (90) - +- HashAggregate (89) - +- Project (88) - +- ShuffledHashJoin Inner BuildRight (87) - :- Exchange (83) - : +- Project (82) - : +- ShuffledHashJoin Inner BuildRight (81) - : :- Exchange (76) - : : +- Project (75) - : : +- ShuffledHashJoin Inner BuildRight (74) - : : :- Exchange (69) - : : : +- Filter (68) - : : : +- Scan parquet (67) - : : +- Exchange (73) - : : +- Project (72) - : : +- Filter (71) - : : +- Scan parquet (70) - : +- Exchange (80) - : +- Project (79) - : +- Filter (78) - : +- Scan parquet (77) - +- Exchange (86) - +- Filter (85) - +- Scan parquet (84) + TakeOrderedAndProject (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- ShuffledHashJoin Inner BuildRight (88) + :- Exchange (84) + : +- Project (83) + : +- ShuffledHashJoin Inner BuildRight (82) + : :- Exchange (77) + : : +- Project (76) + : : +- ShuffledHashJoin Inner BuildRight (75) + : : :- Exchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- Exchange (74) + : : +- Project (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (81) + : +- Project (80) + : +- Filter (79) + : +- Scan parquet (78) + +- Exchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -315,167 +316,171 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(58) ColumnarExchange +(58) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(59) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(60) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(60) InputAdapter +(61) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(61) InputIteratorTransformer +(62) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(62) RegularHashAggregateExecTransformer +(63) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(63) ProjectExecTransformer +(64) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(65) TakeOrderedAndProjectExecTransformer +(66) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(66) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) Scan parquet +(68) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(68) Filter +(69) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(69) Exchange +(70) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(70) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(72) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(73) Exchange +(74) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) ShuffledHashJoin +(75) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(75) Project +(76) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(76) Exchange +(77) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Scan parquet +(78) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(78) Filter +(79) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(79) Project +(80) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(80) Exchange +(81) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(82) Project +(83) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(83) Exchange +(84) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(86) Exchange +(87) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) ShuffledHashJoin +(88) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) Project +(89) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(89) HashAggregate +(90) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(90) Exchange +(91) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(92) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(92) TakeOrderedAndProject +(93) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(93) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt index b3fb729d8d96..8d17beb8c0a9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt @@ -1,70 +1,71 @@ == Physical Plan == -AdaptiveSparkPlan (77) +AdaptiveSparkPlan (78) +- == Final Plan == - VeloxColumnarToRowExec (55) - +- ^ SortExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- VeloxAppendBatches (48) - +- ^ FilterExecTransformer (46) - +- ^ RegularHashAggregateExecTransformer (45) - +- ^ InputIteratorTransformer (44) - +- ShuffleQueryStage (42), Statistics(X) - +- ColumnarExchange (41) - +- ^ ProjectExecTransformer (39) - +- ^ FlushableHashAggregateExecTransformer (38) - +- ^ ProjectExecTransformer (37) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - :- ^ InputIteratorTransformer (26) - : +- ShuffleQueryStage (24), Statistics(X) - : +- ColumnarExchange (23) - : +- VeloxAppendBatches (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : :- ^ InputIteratorTransformer (9) - : : +- ShuffleQueryStage (7), Statistics(X) - : : +- ColumnarExchange (6) - : : +- VeloxAppendBatches (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (18) - : +- ShuffleQueryStage (16), Statistics(X) - : +- ColumnarExchange (15) - : +- VeloxAppendBatches (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ NoopFilter (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- VeloxAppendBatches (31) - +- ^ ProjectExecTransformer (29) - +- ^ NoopFilter (28) - +- ^ Scan parquet (27) + VeloxColumnarToRowExec (56) + +- ^ SortExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ShuffleQueryStage (51), Statistics(X) + +- ColumnarExchange (50) + +- VeloxAppendBatches (49) + +- ^ FilterExecTransformer (47) + +- ^ RegularHashAggregateExecTransformer (46) + +- ^ InputIteratorTransformer (45) + +- ShuffleQueryStage (43), Statistics(X) + +- ColumnarExchange (42) + +- VeloxAppendBatches (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ NoopFilter (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (76) - +- Exchange (75) - +- Filter (74) - +- HashAggregate (73) - +- Exchange (72) - +- HashAggregate (71) - +- Project (70) - +- ShuffledHashJoin Inner BuildRight (69) - :- Exchange (64) - : +- Project (63) - : +- ShuffledHashJoin Inner BuildRight (62) - : :- Exchange (58) - : : +- Filter (57) - : : +- Scan parquet (56) - : +- Exchange (61) - : +- Filter (60) - : +- Scan parquet (59) - +- Exchange (68) - +- Project (67) - +- Filter (66) - +- Scan parquet (65) + Sort (77) + +- Exchange (76) + +- Filter (75) + +- HashAggregate (74) + +- Exchange (73) + +- HashAggregate (72) + +- Project (71) + +- ShuffledHashJoin Inner BuildRight (70) + :- Exchange (65) + : +- Project (64) + : +- ShuffledHashJoin Inner BuildRight (63) + : :- Exchange (59) + : : +- Filter (58) + : : +- Scan parquet (57) + : +- Exchange (62) + : +- Filter (61) + : +- Scan parquet (60) + +- Exchange (69) + +- Project (68) + +- Filter (67) + +- Scan parquet (66) (1) Scan parquet @@ -233,432 +234,436 @@ Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(41) ColumnarExchange +(41) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(42) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(42) ShuffleQueryStage +(43) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(43) InputAdapter +(44) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(44) InputIteratorTransformer +(45) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(45) RegularHashAggregateExecTransformer +(46) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(46) FilterExecTransformer +(47) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(47) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(48) VeloxAppendBatches +(49) VeloxAppendBatches Input [2]: [ps_partkey#X, value#X] Arguments: X -(49) ColumnarExchange +(50) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(51) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(51) InputAdapter +(52) InputAdapter Input [2]: [ps_partkey#X, value#X] -(52) InputIteratorTransformer +(53) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(53) SortExecTransformer +(54) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(54) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(55) VeloxColumnarToRowExec +(56) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(56) Scan parquet +(57) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(57) Filter +(58) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(58) Exchange +(59) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) Scan parquet +(60) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(60) Filter +(61) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(61) Exchange +(62) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) ShuffledHashJoin +(63) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(63) Project +(64) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(64) Exchange +(65) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) Scan parquet +(66) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(67) Project +(68) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(68) Exchange +(69) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(69) ShuffledHashJoin +(70) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(70) Project +(71) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(71) HashAggregate +(72) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(72) Exchange +(73) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(73) HashAggregate +(74) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(74) Filter +(75) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(75) Exchange +(76) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) Sort +(77) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(77) AdaptiveSparkPlan +(78) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (127) +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (109) - +- ^ ProjectExecTransformer (107) - +- ^ RegularHashAggregateExecTransformer (106) - +- ^ RegularHashAggregateExecTransformer (105) - +- ^ ProjectExecTransformer (104) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (103) - :- ^ InputIteratorTransformer (98) - : +- ShuffleQueryStage (96), Statistics(X) - : +- ColumnarExchange (95) - : +- VeloxAppendBatches (94) - : +- ^ ProjectExecTransformer (92) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (91) - : :- ^ InputIteratorTransformer (86) - : : +- ShuffleQueryStage (84), Statistics(X) - : : +- ColumnarExchange (83) - : : +- VeloxAppendBatches (82) - : : +- ^ ProjectExecTransformer (80) - : : +- ^ NoopFilter (79) - : : +- ^ Scan parquet (78) - : +- ^ InputIteratorTransformer (90) - : +- ShuffleQueryStage (88), Statistics(X) - : +- ReusedExchange (87) - +- ^ InputIteratorTransformer (102) - +- ShuffleQueryStage (100), Statistics(X) - +- ReusedExchange (99) + VeloxColumnarToRowExec (110) + +- ^ ProjectExecTransformer (108) + +- ^ RegularHashAggregateExecTransformer (107) + +- ^ RegularHashAggregateExecTransformer (106) + +- ^ ProjectExecTransformer (105) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + :- ^ InputIteratorTransformer (99) + : +- ShuffleQueryStage (97), Statistics(X) + : +- ColumnarExchange (96) + : +- VeloxAppendBatches (95) + : +- ^ ProjectExecTransformer (93) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) + : :- ^ InputIteratorTransformer (87) + : : +- ShuffleQueryStage (85), Statistics(X) + : : +- ColumnarExchange (84) + : : +- VeloxAppendBatches (83) + : : +- ^ ProjectExecTransformer (81) + : : +- ^ NoopFilter (80) + : : +- ^ Scan parquet (79) + : +- ^ InputIteratorTransformer (91) + : +- ShuffleQueryStage (89), Statistics(X) + : +- ReusedExchange (88) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ReusedExchange (100) +- == Initial Plan == - HashAggregate (126) - +- HashAggregate (125) - +- Project (124) - +- ShuffledHashJoin Inner BuildRight (123) - :- Exchange (118) - : +- Project (117) - : +- ShuffledHashJoin Inner BuildRight (116) - : :- Exchange (112) - : : +- Filter (111) - : : +- Scan parquet (110) - : +- Exchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- Exchange (122) - +- Project (121) - +- Filter (120) - +- Scan parquet (119) - - -(78) Scan parquet + HashAggregate (127) + +- HashAggregate (126) + +- Project (125) + +- ShuffledHashJoin Inner BuildRight (124) + :- Exchange (119) + : +- Project (118) + : +- ShuffledHashJoin Inner BuildRight (117) + : :- Exchange (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Exchange (116) + : +- Filter (115) + : +- Scan parquet (114) + +- Exchange (123) + +- Project (122) + +- Filter (121) + +- Scan parquet (120) + + +(79) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(79) NoopFilter +(80) NoopFilter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(80) ProjectExecTransformer +(81) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(81) WholeStageCodegenTransformer (X) +(82) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(82) VeloxAppendBatches +(83) VeloxAppendBatches Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(83) ColumnarExchange +(84) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(84) ShuffleQueryStage +(85) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(85) InputAdapter +(86) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(86) InputIteratorTransformer +(87) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(87) ReusedExchange [Reuses operator id: 15] +(88) ReusedExchange [Reuses operator id: 15] Output [2]: [s_suppkey#X, s_nationkey#X] -(88) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(89) InputAdapter +(90) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(90) InputIteratorTransformer +(91) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(91) ShuffledHashJoinExecTransformer +(92) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(92) ProjectExecTransformer +(93) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(93) WholeStageCodegenTransformer (X) +(94) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(94) VeloxAppendBatches +(95) VeloxAppendBatches Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(95) ColumnarExchange +(96) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(97) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(97) InputAdapter +(98) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(98) InputIteratorTransformer +(99) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(99) ReusedExchange [Reuses operator id: 32] +(100) ReusedExchange [Reuses operator id: 32] Output [1]: [n_nationkey#X] -(100) ShuffleQueryStage +(101) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(101) InputAdapter +(102) InputAdapter Input [1]: [n_nationkey#X] -(102) InputIteratorTransformer +(103) InputIteratorTransformer Input [1]: [n_nationkey#X] -(103) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(104) ProjectExecTransformer +(105) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(105) RegularHashAggregateExecTransformer +(106) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(106) RegularHashAggregateExecTransformer +(107) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(107) ProjectExecTransformer +(108) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(108) WholeStageCodegenTransformer (X) +(109) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(109) VeloxColumnarToRowExec +(110) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(110) Scan parquet +(111) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(112) Exchange +(113) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) Scan parquet +(114) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(114) Filter +(115) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(115) Exchange +(116) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(117) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(117) Project +(118) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(118) Exchange +(119) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) Scan parquet +(120) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(120) Filter +(121) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(121) Project +(122) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(122) Exchange +(123) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) ShuffledHashJoin +(124) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(124) Project +(125) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(125) HashAggregate +(126) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(126) HashAggregate +(127) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(127) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt index c9a5452d9aa0..dd1259eb8876 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt @@ -1,49 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32), Statistics(X) - +- ColumnarExchange (31) - +- VeloxAppendBatches (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7), Statistics(X) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- VeloxAppendBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- Project (46) - +- ShuffledHashJoin Inner BuildLeft (45) - :- Exchange (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Exchange (44) - +- Project (43) - +- Filter (42) - +- Scan parquet (41) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin Inner BuildLeft (46) + :- Exchange (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -144,129 +145,133 @@ Input [3]: [l_shipmode#X, sum#X, sum#X] Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(24) ColumnarExchange +(24) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(25) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(28) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(29) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(30) VeloxAppendBatches +(31) VeloxAppendBatches Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(31) ColumnarExchange +(32) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) SortExecTransformer +(36) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(37) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(38) Scan parquet +(39) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(40) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(42) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(42) Filter +(43) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(43) Project +(44) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(44) Exchange +(45) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(46) Project +(47) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(47) HashAggregate +(48) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(48) Exchange +(49) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(50) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(50) Exchange +(51) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(52) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(52) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt index 0a402001c23b..d43ad2a9c271 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (55) +AdaptiveSparkPlan (56) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- ^ SortExecTransformer (37) - +- ^ InputIteratorTransformer (36) - +- ShuffleQueryStage (34), Statistics(X) - +- ColumnarExchange (33) - +- VeloxAppendBatches (32) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- VeloxAppendBatches (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (54) - +- Exchange (53) - +- HashAggregate (52) - +- Exchange (51) - +- HashAggregate (50) - +- HashAggregate (49) - +- HashAggregate (48) - +- Project (47) - +- ShuffledHashJoin LeftOuter BuildRight (46) - :- Exchange (41) - : +- Scan parquet (40) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftOuter BuildRight (47) + :- Exchange (42) + : +- Scan parquet (41) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -160,138 +161,142 @@ Input [2]: [c_count#X, count#X] Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [2]: [c_count#X, count#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(32) VeloxAppendBatches +(33) VeloxAppendBatches Input [2]: [c_count#X, custdist#X] Arguments: X -(33) ColumnarExchange +(34) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(34) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(35) InputAdapter +(36) InputAdapter Input [2]: [c_count#X, custdist#X] -(36) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(37) SortExecTransformer +(38) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(38) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(39) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(40) Scan parquet +(41) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(41) Exchange +(42) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Scan parquet +(43) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(43) Filter +(44) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(44) Project +(45) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(45) Exchange +(46) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(47) Project +(48) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(48) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(49) HashAggregate +(50) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(50) HashAggregate +(51) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(51) Exchange +(52) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) HashAggregate +(53) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(53) Exchange +(54) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Sort +(55) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(55) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt index 7f0635c0d9a4..60521d6c62cd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt @@ -1,13 +1,13 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- AQEShuffleRead (28) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- VeloxAppendBatches (25) - +- ^ ProjectExecTransformer (23) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (22) + VeloxColumnarToRowExec (30) + +- AQEShuffleRead (29) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (23) :- ^ InputIteratorTransformer (9) : +- ShuffleQueryStage (7), Statistics(X) : +- ColumnarExchange (6) @@ -15,31 +15,32 @@ AdaptiveSparkPlan (44) : +- ^ ProjectExecTransformer (3) : +- ^ NoopFilter (2) : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- Project (41) - +- ShuffledHashJoin Inner BuildLeft (40) - :- Exchange (32) - : +- Filter (31) - : +- Scan parquet (30) - +- Filter (39) - +- HashAggregate (38) - +- Exchange (37) - +- HashAggregate (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (44) + +- Exchange (43) + +- Project (42) + +- ShuffledHashJoin Inner BuildLeft (41) + :- Exchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Filter (40) + +- HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -109,288 +110,297 @@ Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(16) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(21) FilterExecTransformer +(22) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(22) ShuffledHashJoinExecTransformer +(23) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(23) ProjectExecTransformer +(24) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(24) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(25) VeloxAppendBatches +(26) VeloxAppendBatches Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) ColumnarExchange +(27) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(28) AQEShuffleRead +(29) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(30) Scan parquet +(31) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(32) Exchange +(33) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(33) Scan parquet +(34) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(34) Filter +(35) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(35) Project +(36) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(36) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(37) Exchange +(38) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) HashAggregate +(39) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(39) Filter +(40) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(40) ShuffledHashJoin +(41) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(41) Project +(42) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(42) Exchange +(43) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(44) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 21 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (69) +Subquery:1 Hosting operator id = 22 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (71) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ RegularHashAggregateExecTransformer (58) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ ProjectExecTransformer (56) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ InputIteratorTransformer (54) - +- ShuffleQueryStage (52), Statistics(X) - +- ColumnarExchange (51) - +- ^ ProjectExecTransformer (49) - +- ^ FlushableHashAggregateExecTransformer (48) - +- ^ ProjectExecTransformer (47) - +- ^ NoopFilter (46) - +- ^ Scan parquet (45) + VeloxColumnarToRowExec (62) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ ProjectExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ NoopFilter (47) + +- ^ Scan parquet (46) +- == Initial Plan == - HashAggregate (68) - +- HashAggregate (67) - +- HashAggregate (66) - +- Exchange (65) - +- HashAggregate (64) - +- Project (63) - +- Filter (62) - +- Scan parquet (61) + HashAggregate (70) + +- HashAggregate (69) + +- HashAggregate (68) + +- Exchange (67) + +- HashAggregate (66) + +- Project (65) + +- Filter (64) + +- Scan parquet (63) -(45) Scan parquet +(46) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(46) NoopFilter +(47) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(47) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(48) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(49) ProjectExecTransformer +(50) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(50) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(51) ColumnarExchange +(52) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(53) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(54) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(53) InputAdapter +(55) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(54) InputIteratorTransformer +(56) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(55) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(56) ProjectExecTransformer +(58) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(57) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(58) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(59) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(60) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(61) Scan parquet +(63) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(62) Filter +(64) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(63) Project +(65) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(64) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(65) Exchange +(67) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) HashAggregate +(68) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(67) HashAggregate +(69) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(68) HashAggregate +(70) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(69) AdaptiveSparkPlan +(71) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt index b9a3fb8dadd0..029516a40506 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt @@ -1,62 +1,64 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (45) - +- ^ SortExecTransformer (43) - +- ^ InputIteratorTransformer (42) - +- ShuffleQueryStage (40), Statistics(X) - +- ColumnarExchange (39) - +- VeloxAppendBatches (38) - +- ^ RegularHashAggregateExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ ProjectExecTransformer (30) - +- ^ FlushableHashAggregateExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7), Statistics(X) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- VeloxAppendBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (47) + +- ^ SortExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42), Statistics(X) + +- ColumnarExchange (41) + +- VeloxAppendBatches (40) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ ProjectExecTransformer (31) + +- ^ FlushableHashAggregateExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (66) - +- Exchange (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- Project (58) - +- ShuffledHashJoin Inner BuildRight (57) - :- Exchange (53) - : +- BroadcastHashJoin LeftAnti BuildRight (52) - : :- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (51) - : +- Project (50) - : +- Filter (49) - : +- Scan parquet (48) - +- Exchange (56) - +- Filter (55) - +- Scan parquet (54) + Sort (68) + +- Exchange (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- HashAggregate (63) + +- Exchange (62) + +- HashAggregate (61) + +- Project (60) + +- ShuffledHashJoin Inner BuildRight (59) + :- Exchange (55) + : +- BroadcastHashJoin LeftAnti BuildRight (54) + : :- Filter (49) + : : +- Scan parquet (48) + : +- BroadcastExchange (53) + : +- Project (52) + : +- Filter (51) + : +- Scan parquet (50) + +- Exchange (58) + +- Filter (57) + +- Scan parquet (56) (1) Scan parquet @@ -157,203 +159,211 @@ Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(24) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(28) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(29) FlushableHashAggregateExecTransformer +(30) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(30) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(32) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(34) InputAdapter +(36) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(35) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(36) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(37) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(38) VeloxAppendBatches +(40) VeloxAppendBatches Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(39) ColumnarExchange +(41) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(42) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(41) InputAdapter +(43) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(42) InputIteratorTransformer +(44) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) SortExecTransformer +(45) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(44) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(45) VeloxColumnarToRowExec +(47) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(46) Scan parquet +(48) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(47) Filter +(49) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(48) Scan parquet +(50) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(49) Filter +(51) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(50) Project +(52) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(51) BroadcastExchange +(53) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(52) BroadcastHashJoin +(54) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(53) Exchange +(55) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(56) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(56) Exchange +(58) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(59) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(58) Project +(60) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(59) HashAggregate +(61) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(60) Exchange +(62) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(63) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(62) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(63) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(66) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(65) Exchange +(67) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) Sort +(68) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(67) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt index 6206907a8588..e2d1503799a9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt @@ -1,12 +1,12 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- ^ ProjectExecTransformer (37) - +- ^ RegularHashAggregateExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ ProjectExecTransformer (34) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (33) + VeloxColumnarToRowExec (40) + +- ^ ProjectExecTransformer (38) + +- ^ RegularHashAggregateExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ ProjectExecTransformer (35) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (34) :- ^ ProjectExecTransformer (20) : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) : :- ^ InputIteratorTransformer (9) @@ -23,36 +23,37 @@ AdaptiveSparkPlan (59) : +- ^ ProjectExecTransformer (12) : +- ^ NoopFilter (11) : +- ^ Scan parquet (10) - +- ^ FilterExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ NoopFilter (22) - +- ^ Scan parquet (21) + +- ^ FilterExecTransformer (33) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ NoopFilter (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (58) - +- HashAggregate (57) - +- Project (56) - +- ShuffledHashJoin Inner BuildRight (55) - :- Project (48) - : +- ShuffledHashJoin Inner BuildRight (47) - : :- Exchange (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Project (45) - : +- Filter (44) - : +- Scan parquet (43) - +- Filter (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- Filter (50) - +- Scan parquet (49) + HashAggregate (59) + +- HashAggregate (58) + +- Project (57) + +- ShuffledHashJoin Inner BuildRight (56) + :- Project (49) + : +- ShuffledHashJoin Inner BuildRight (48) + : :- Exchange (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Exchange (47) + : +- Project (46) + : +- Filter (45) + : +- Scan parquet (44) + +- Filter (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- Filter (51) + +- Scan parquet (50) (1) Scan parquet @@ -164,168 +165,172 @@ Input [3]: [l_partkey#X, sum#X, count#X] Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(32) FilterExecTransformer +(33) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(33) ShuffledHashJoinExecTransformer +(34) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(34) ProjectExecTransformer +(35) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(35) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(36) RegularHashAggregateExecTransformer +(37) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(37) ProjectExecTransformer +(38) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(38) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(39) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(40) Scan parquet +(41) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(41) Filter +(42) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(42) Exchange +(43) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Scan parquet +(44) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(45) Project +(46) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(46) Exchange +(47) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(48) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(48) Project +(49) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(49) Scan parquet +(50) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(50) Filter +(51) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(51) HashAggregate +(52) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(52) Exchange +(53) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(54) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(54) Filter +(55) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(55) ShuffledHashJoin +(56) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(56) Project +(57) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(57) HashAggregate +(58) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(58) HashAggregate +(59) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] -(59) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt index 038422e0fecf..a0e052432bb3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt @@ -1,18 +1,18 @@ == Physical Plan == -AdaptiveSparkPlan (102) +AdaptiveSparkPlan (103) +- == Final Plan == - VeloxColumnarToRowExec (69) - +- TakeOrderedAndProjectExecTransformer (68) - +- ^ RegularHashAggregateExecTransformer (66) - +- ^ RegularHashAggregateExecTransformer (65) - +- ^ ProjectExecTransformer (64) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (63) - :- ^ InputIteratorTransformer (45) - : +- ShuffleQueryStage (43), Statistics(X) - : +- ColumnarExchange (42) - : +- VeloxAppendBatches (41) - : +- ^ ProjectExecTransformer (39) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (38) + VeloxColumnarToRowExec (70) + +- TakeOrderedAndProjectExecTransformer (69) + +- ^ RegularHashAggregateExecTransformer (67) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ ProjectExecTransformer (65) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (64) + :- ^ InputIteratorTransformer (46) + : +- ShuffleQueryStage (44), Statistics(X) + : +- ColumnarExchange (43) + : +- VeloxAppendBatches (42) + : +- ^ ProjectExecTransformer (40) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (39) : :- ^ InputIteratorTransformer (9) : : +- ShuffleQueryStage (7), Statistics(X) : : +- ColumnarExchange (6) @@ -20,12 +20,12 @@ AdaptiveSparkPlan (102) : : +- ^ ProjectExecTransformer (3) : : +- ^ NoopFilter (2) : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (37) - : +- ShuffleQueryStage (35), Statistics(X) - : +- ColumnarExchange (34) - : +- VeloxAppendBatches (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (30) + : +- ^ InputIteratorTransformer (38) + : +- ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- VeloxAppendBatches (34) + : +- ^ ProjectExecTransformer (32) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (31) : :- ^ InputIteratorTransformer (18) : : +- ShuffleQueryStage (16), Statistics(X) : : +- ColumnarExchange (15) @@ -33,62 +33,63 @@ AdaptiveSparkPlan (102) : : +- ^ ProjectExecTransformer (12) : : +- ^ NoopFilter (11) : : +- ^ Scan parquet (10) - : +- ^ ProjectExecTransformer (29) - : +- ^ FilterExecTransformer (28) - : +- ^ RegularHashAggregateExecTransformer (27) - : +- ^ InputIteratorTransformer (26) - : +- ShuffleQueryStage (24), Statistics(X) - : +- ColumnarExchange (23) - : +- ^ ProjectExecTransformer (21) - : +- ^ FlushableHashAggregateExecTransformer (20) - : +- ^ Scan parquet (19) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - :- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- VeloxAppendBatches (50) - : +- ^ ProjectExecTransformer (48) - : +- ^ NoopFilter (47) - : +- ^ Scan parquet (46) - +- ^ ProjectExecTransformer (61) - +- ^ FilterExecTransformer (60) - +- ^ RegularHashAggregateExecTransformer (59) - +- ^ InputIteratorTransformer (58) - +- ShuffleQueryStage (56), Statistics(X) - +- ReusedExchange (55) + : +- ^ ProjectExecTransformer (30) + : +- ^ FilterExecTransformer (29) + : +- ^ RegularHashAggregateExecTransformer (28) + : +- ^ InputIteratorTransformer (27) + : +- ShuffleQueryStage (25), Statistics(X) + : +- ColumnarExchange (24) + : +- VeloxAppendBatches (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (63) + :- ^ InputIteratorTransformer (55) + : +- ShuffleQueryStage (53), Statistics(X) + : +- ColumnarExchange (52) + : +- VeloxAppendBatches (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ NoopFilter (48) + : +- ^ Scan parquet (47) + +- ^ ProjectExecTransformer (62) + +- ^ FilterExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57), Statistics(X) + +- ReusedExchange (56) +- == Initial Plan == - TakeOrderedAndProject (101) - +- HashAggregate (100) - +- HashAggregate (99) - +- Project (98) - +- ShuffledHashJoin Inner BuildRight (97) - :- Exchange (86) - : +- Project (85) - : +- ShuffledHashJoin Inner BuildLeft (84) - : :- Exchange (72) - : : +- Filter (71) - : : +- Scan parquet (70) - : +- Exchange (83) - : +- ShuffledHashJoin LeftSemi BuildRight (82) - : :- Exchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- Project (81) - : +- Filter (80) - : +- HashAggregate (79) - : +- Exchange (78) - : +- HashAggregate (77) - : +- Scan parquet (76) - +- ShuffledHashJoin LeftSemi BuildRight (96) - :- Exchange (89) - : +- Filter (88) - : +- Scan parquet (87) - +- Project (95) - +- Filter (94) - +- HashAggregate (93) - +- Exchange (92) - +- HashAggregate (91) - +- Scan parquet (90) + TakeOrderedAndProject (102) + +- HashAggregate (101) + +- HashAggregate (100) + +- Project (99) + +- ShuffledHashJoin Inner BuildRight (98) + :- Exchange (87) + : +- Project (86) + : +- ShuffledHashJoin Inner BuildLeft (85) + : :- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (84) + : +- ShuffledHashJoin LeftSemi BuildRight (83) + : :- Exchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Project (82) + : +- Filter (81) + : +- HashAggregate (80) + : +- Exchange (79) + : +- HashAggregate (78) + : +- Scan parquet (77) + +- ShuffledHashJoin LeftSemi BuildRight (97) + :- Exchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- Project (96) + +- Filter (95) + +- HashAggregate (94) + +- Exchange (93) + +- HashAggregate (92) + +- Scan parquet (91) (1) Scan parquet @@ -186,364 +187,368 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(28) FilterExecTransformer +(29) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(29) ProjectExecTransformer +(30) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(30) ShuffledHashJoinExecTransformer +(31) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(33) VeloxAppendBatches +(34) VeloxAppendBatches Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(34) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(35) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(36) InputAdapter +(37) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(38) ShuffledHashJoinExecTransformer +(39) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(39) ProjectExecTransformer +(40) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(40) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(41) VeloxAppendBatches +(42) VeloxAppendBatches Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(42) ColumnarExchange +(43) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(43) ShuffleQueryStage +(44) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(44) InputAdapter +(45) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(45) InputIteratorTransformer +(46) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(46) Scan parquet +(47) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(47) NoopFilter +(48) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(48) ProjectExecTransformer +(49) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(49) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(50) VeloxAppendBatches +(51) VeloxAppendBatches Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: X -(51) ColumnarExchange +(52) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(53) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(53) InputAdapter +(54) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(54) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(55) ReusedExchange [Reuses operator id: 23] +(56) ReusedExchange [Reuses operator id: 24] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(56) ShuffleQueryStage +(57) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(57) InputAdapter +(58) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(58) InputIteratorTransformer +(59) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(59) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(60) FilterExecTransformer +(61) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(61) ProjectExecTransformer +(62) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(62) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(63) ShuffledHashJoinExecTransformer +(64) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(64) ProjectExecTransformer +(65) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(65) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(66) RegularHashAggregateExecTransformer +(67) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(67) WholeStageCodegenTransformer (X) +(68) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(68) TakeOrderedAndProjectExecTransformer +(69) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(69) VeloxColumnarToRowExec +(70) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(70) Scan parquet +(71) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(72) Exchange +(73) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(73) Scan parquet +(74) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(75) Exchange +(76) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) Scan parquet +(77) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(77) HashAggregate +(78) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(78) Exchange +(79) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) HashAggregate +(80) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(80) Filter +(81) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(81) Project +(82) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(82) ShuffledHashJoin +(83) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(83) Exchange +(84) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) ShuffledHashJoin +(85) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(85) Project +(86) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(86) Exchange +(87) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Scan parquet +(88) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(88) Filter +(89) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(89) Exchange +(90) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(91) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(91) HashAggregate +(92) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(92) Exchange +(93) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) HashAggregate +(94) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(94) Filter +(95) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(95) Project +(96) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(96) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(97) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(98) Project +(99) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(99) HashAggregate +(100) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(100) HashAggregate +(101) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(101) TakeOrderedAndProject +(102) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(102) AdaptiveSparkPlan +(103) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt index d2c18146af4f..bd77a7f7f043 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt @@ -1,112 +1,113 @@ == Physical Plan == -AdaptiveSparkPlan (127) +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ RegularHashAggregateExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ ProjectExecTransformer (82) - +- ^ FlushableHashAggregateExecTransformer (81) - +- ^ ProjectExecTransformer (80) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) - :- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67), Statistics(X) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50), Statistics(X) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) - : : :- ^ InputIteratorTransformer (9) - : : : +- ShuffleQueryStage (7), Statistics(X) - : : : +- ColumnarExchange (6) - : : : +- VeloxAppendBatches (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (44) - : : +- ShuffleQueryStage (42), Statistics(X) - : : +- ColumnarExchange (41) - : : +- VeloxAppendBatches (40) - : : +- ^ ProjectExecTransformer (38) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) - : : : :- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (26) - : : : +- ShuffleQueryStage (24), Statistics(X) - : : : +- ColumnarExchange (23) - : : : +- VeloxAppendBatches (22) - : : : +- ^ ProjectExecTransformer (20) - : : : +- ^ Scan parquet (19) - : : +- ^ InputIteratorTransformer (36) - : : +- ShuffleQueryStage (34), Statistics(X) - : : +- ColumnarExchange (33) - : : +- VeloxAppendBatches (32) - : : +- ^ ProjectExecTransformer (30) - : : +- ^ NoopFilter (29) - : : +- ^ Scan parquet (28) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- VeloxAppendBatches (57) - : +- ^ ProjectExecTransformer (55) - : +- ^ NoopFilter (54) - : +- ^ Scan parquet (53) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- VeloxAppendBatches (74) - +- ^ ProjectExecTransformer (72) - +- ^ NoopFilter (71) - +- ^ Scan parquet (70) + VeloxColumnarToRowExec (91) + +- ^ RegularHashAggregateExecTransformer (89) + +- ^ InputIteratorTransformer (88) + +- ShuffleQueryStage (86), Statistics(X) + +- ColumnarExchange (85) + +- VeloxAppendBatches (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24), Statistics(X) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34), Statistics(X) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ NoopFilter (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ NoopFilter (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ NoopFilter (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (116) - : +- Project (115) - : +- ShuffledHashJoin Inner BuildRight (114) - : :- Exchange (109) - : : +- Project (108) - : : +- ShuffledHashJoin Inner BuildLeft (107) - : : :- Exchange (93) - : : : +- Filter (92) - : : : +- Scan parquet (91) - : : +- Exchange (106) - : : +- ShuffledHashJoin LeftAnti BuildRight (105) - : : :- ShuffledHashJoin LeftSemi BuildRight (100) - : : : :- Exchange (97) - : : : : +- Project (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (99) - : : : +- Scan parquet (98) - : : +- Exchange (104) - : : +- Project (103) - : : +- Filter (102) - : : +- Scan parquet (101) - : +- Exchange (113) - : +- Project (112) - : +- Filter (111) - : +- Scan parquet (110) - +- Exchange (120) - +- Project (119) - +- Filter (118) - +- Scan parquet (117) + TakeOrderedAndProject (127) + +- HashAggregate (126) + +- Exchange (125) + +- HashAggregate (124) + +- Project (123) + +- ShuffledHashJoin Inner BuildRight (122) + :- Exchange (117) + : +- Project (116) + : +- ShuffledHashJoin Inner BuildRight (115) + : :- Exchange (110) + : : +- Project (109) + : : +- ShuffledHashJoin Inner BuildLeft (108) + : : :- Exchange (94) + : : : +- Filter (93) + : : : +- Scan parquet (92) + : : +- Exchange (107) + : : +- ShuffledHashJoin LeftAnti BuildRight (106) + : : :- ShuffledHashJoin LeftSemi BuildRight (101) + : : : :- Exchange (98) + : : : : +- Project (97) + : : : : +- Filter (96) + : : : : +- Scan parquet (95) + : : : +- Exchange (100) + : : : +- Scan parquet (99) + : : +- Exchange (105) + : : +- Project (104) + : : +- Filter (103) + : : +- Scan parquet (102) + : +- Exchange (114) + : +- Project (113) + : +- Filter (112) + : +- Scan parquet (111) + +- Exchange (121) + +- Project (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -448,206 +449,210 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(84) ColumnarExchange +(84) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(85) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(86) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(86) InputAdapter +(87) InputAdapter Input [2]: [s_name#X, count#X] -(87) InputIteratorTransformer +(88) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(88) RegularHashAggregateExecTransformer +(89) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(89) WholeStageCodegenTransformer (X) +(90) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(90) VeloxColumnarToRowExec +(91) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(91) Scan parquet +(92) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(93) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(94) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(95) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(96) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Project +(97) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(97) Exchange +(98) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(99) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(99) Exchange +(100) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) ShuffledHashJoin +(101) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(101) Scan parquet +(102) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(102) Filter +(103) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(103) Project +(104) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(104) Exchange +(105) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) ShuffledHashJoin +(106) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(106) Exchange +(107) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(108) Project +(109) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(109) Exchange +(110) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(112) Project +(113) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(113) Exchange +(114) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(115) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(115) Project +(116) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(116) Exchange +(117) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(118) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(118) Filter +(119) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(119) Project +(120) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(120) Exchange +(121) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(122) Project +(123) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(123) HashAggregate +(124) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(124) Exchange +(125) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(126) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(126) TakeOrderedAndProject +(127) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(127) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt index ea3839718ac1..7cf55b4c0f2d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt @@ -1,46 +1,47 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (50) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- VeloxAppendBatches (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7), Statistics(X) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ ProjectExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin LeftAnti BuildRight (42) - :- Exchange (39) - : +- Filter (38) - : +- Scan parquet (37) - +- Exchange (41) - +- Scan parquet (40) + Sort (49) + +- Exchange (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftAnti BuildRight (43) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (42) + +- Scan parquet (41) (1) Scan parquet @@ -136,234 +137,243 @@ Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(29) VeloxAppendBatches +(30) VeloxAppendBatches Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(30) ColumnarExchange +(31) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(32) InputAdapter +(33) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(33) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) SortExecTransformer +(35) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(35) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(36) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(37) Scan parquet +(38) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(38) Filter +(39) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(39) Exchange +(40) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) Scan parquet +(41) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(41) Exchange +(42) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(44) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(45) Exchange +(46) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(47) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(47) Exchange +(48) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(49) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(50) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (68) +AdaptiveSparkPlan (70) +- == Final Plan == - VeloxColumnarToRowExec (61) - +- ^ RegularHashAggregateExecTransformer (59) - +- ^ InputIteratorTransformer (58) - +- ShuffleQueryStage (56), Statistics(X) - +- ColumnarExchange (55) - +- ^ FlushableHashAggregateExecTransformer (53) - +- ^ ProjectExecTransformer (52) - +- ^ NoopFilter (51) - +- ^ Scan parquet (50) + VeloxColumnarToRowExec (63) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ InputIteratorTransformer (60) + +- ShuffleQueryStage (58), Statistics(X) + +- ColumnarExchange (57) + +- VeloxAppendBatches (56) + +- ^ FlushableHashAggregateExecTransformer (54) + +- ^ ProjectExecTransformer (53) + +- ^ NoopFilter (52) + +- ^ Scan parquet (51) +- == Initial Plan == - HashAggregate (67) - +- Exchange (66) - +- HashAggregate (65) - +- Project (64) - +- Filter (63) - +- Scan parquet (62) + HashAggregate (69) + +- Exchange (68) + +- HashAggregate (67) + +- Project (66) + +- Filter (65) + +- Scan parquet (64) -(50) Scan parquet +(51) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(51) NoopFilter +(52) NoopFilter Input [2]: [c_phone#X, c_acctbal#X] Arguments: [c_phone#X, c_acctbal#X] -(52) ProjectExecTransformer +(53) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(53) FlushableHashAggregateExecTransformer +(54) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(54) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(55) ColumnarExchange +(56) VeloxAppendBatches +Input [2]: [sum#X, count#X] +Arguments: X + +(57) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(56) ShuffleQueryStage +(58) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(57) InputAdapter +(59) InputAdapter Input [2]: [sum#X, count#X] -(58) InputIteratorTransformer +(60) InputIteratorTransformer Input [2]: [sum#X, count#X] -(59) RegularHashAggregateExecTransformer +(61) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(60) WholeStageCodegenTransformer (X) +(62) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(61) VeloxColumnarToRowExec +(63) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(62) Scan parquet +(64) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(63) Filter +(65) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(64) Project +(66) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(65) HashAggregate +(67) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(66) Exchange +(68) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(67) HashAggregate +(69) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(68) AdaptiveSparkPlan +(70) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt index 054409875b25..1b680584826d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt @@ -1,50 +1,51 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32), Statistics(X) - +- ColumnarExchange (31) - +- VeloxAppendBatches (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7), Statistics(X) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- VeloxAppendBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (52) - +- Exchange (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- ShuffledHashJoin LeftSemi BuildRight (46) - :- Exchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (53) + +- Exchange (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftSemi BuildRight (47) + :- Exchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -145,133 +146,137 @@ Input [2]: [o_orderpriority#X, count#X] Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(24) ColumnarExchange +(24) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(28) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(29) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(30) VeloxAppendBatches +(31) VeloxAppendBatches Input [2]: [o_orderpriority#X, order_count#X] Arguments: X -(31) ColumnarExchange +(32) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(35) SortExecTransformer +(36) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(37) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(41) Exchange +(42) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Scan parquet +(43) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(43) Filter +(44) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(44) Project +(45) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(45) Exchange +(46) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(47) Project +(48) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(48) HashAggregate +(49) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(49) Exchange +(50) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(51) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(51) Exchange +(52) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(53) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt index db0ff094f920..67159dbb648a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt @@ -1,126 +1,127 @@ == Physical Plan == -AdaptiveSparkPlan (145) +AdaptiveSparkPlan (146) +- == Final Plan == - VeloxColumnarToRowExec (105) - +- ^ SortExecTransformer (103) - +- ^ InputIteratorTransformer (102) - +- ShuffleQueryStage (100), Statistics(X) - +- ColumnarExchange (99) - +- VeloxAppendBatches (98) - +- ^ RegularHashAggregateExecTransformer (96) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ColumnarExchange (92) - +- ^ ProjectExecTransformer (90) - +- ^ FlushableHashAggregateExecTransformer (89) - +- ^ ProjectExecTransformer (88) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - :- ^ InputIteratorTransformer (77) - : +- ShuffleQueryStage (75), Statistics(X) - : +- ColumnarExchange (74) - : +- VeloxAppendBatches (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - : :- ^ InputIteratorTransformer (60) - : : +- ShuffleQueryStage (58), Statistics(X) - : : +- ColumnarExchange (57) - : : +- VeloxAppendBatches (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - : : :- ^ InputIteratorTransformer (43) - : : : +- ShuffleQueryStage (41), Statistics(X) - : : : +- ColumnarExchange (40) - : : : +- VeloxAppendBatches (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - : : : :- ^ InputIteratorTransformer (26) - : : : : +- ShuffleQueryStage (24), Statistics(X) - : : : : +- ColumnarExchange (23) - : : : : +- VeloxAppendBatches (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ InputIteratorTransformer (9) - : : : : : +- ShuffleQueryStage (7), Statistics(X) - : : : : : +- ColumnarExchange (6) - : : : : : +- VeloxAppendBatches (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (35) - : : : +- ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- VeloxAppendBatches (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ NoopFilter (28) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50), Statistics(X) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ NoopFilter (45) - : : +- ^ Scan parquet (44) - : +- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67), Statistics(X) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ NoopFilter (62) - : +- ^ Scan parquet (61) - +- ^ InputIteratorTransformer (86) - +- ShuffleQueryStage (84), Statistics(X) - +- ColumnarExchange (83) - +- VeloxAppendBatches (82) - +- ^ ProjectExecTransformer (80) - +- ^ NoopFilter (79) - +- ^ Scan parquet (78) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94), Statistics(X) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (144) - +- Exchange (143) - +- HashAggregate (142) - +- Exchange (141) - +- HashAggregate (140) - +- Project (139) - +- ShuffledHashJoin Inner BuildRight (138) - :- Exchange (133) - : +- Project (132) - : +- ShuffledHashJoin Inner BuildRight (131) - : :- Exchange (127) - : : +- Project (126) - : : +- ShuffledHashJoin Inner BuildRight (125) - : : :- Exchange (121) - : : : +- Project (120) - : : : +- ShuffledHashJoin Inner BuildRight (119) - : : : :- Exchange (115) - : : : : +- Project (114) - : : : : +- ShuffledHashJoin Inner BuildLeft (113) - : : : : :- Exchange (108) - : : : : : +- Filter (107) - : : : : : +- Scan parquet (106) - : : : : +- Exchange (112) - : : : : +- Project (111) - : : : : +- Filter (110) - : : : : +- Scan parquet (109) - : : : +- Exchange (118) - : : : +- Filter (117) - : : : +- Scan parquet (116) - : : +- Exchange (124) - : : +- Filter (123) - : : +- Scan parquet (122) - : +- Exchange (130) - : +- Filter (129) - : +- Scan parquet (128) - +- Exchange (137) - +- Project (136) - +- Filter (135) - +- Scan parquet (134) + Sort (145) + +- Exchange (144) + +- HashAggregate (143) + +- Exchange (142) + +- HashAggregate (141) + +- Project (140) + +- ShuffledHashJoin Inner BuildRight (139) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Project (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (138) + +- Project (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -493,245 +494,249 @@ Input [3]: [n_name#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(92) ColumnarExchange +(92) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(93) ShuffleQueryStage +(94) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(94) InputAdapter +(95) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(95) InputIteratorTransformer +(96) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(96) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(97) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(98) VeloxAppendBatches +(99) VeloxAppendBatches Input [2]: [n_name#X, revenue#X] Arguments: X -(99) ColumnarExchange +(100) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(100) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(101) InputAdapter +(102) InputAdapter Input [2]: [n_name#X, revenue#X] -(102) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(103) SortExecTransformer +(104) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(104) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(105) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(106) Scan parquet +(107) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(108) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(109) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Scan parquet +(110) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(110) Filter +(111) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(111) Project +(112) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(112) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(114) Project +(115) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(115) Exchange +(116) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) Scan parquet +(117) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(117) Filter +(118) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(118) Exchange +(119) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(120) Project +(121) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(121) Exchange +(122) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(123) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(123) Filter +(124) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(124) Exchange +(125) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(126) Project +(127) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(127) Exchange +(128) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(129) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(129) Filter +(130) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(130) Exchange +(131) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(132) Project +(133) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(133) Exchange +(134) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(134) Scan parquet +(135) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(135) Filter +(136) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(136) Project +(137) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(137) Exchange +(138) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(138) ShuffledHashJoin +(139) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(139) Project +(140) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(140) HashAggregate +(141) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(141) Exchange +(142) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) HashAggregate +(143) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(143) Exchange +(144) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(144) Sort +(145) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(145) AdaptiveSparkPlan +(146) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt index 30e1b3fe013b..fa9c936a0ca1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt @@ -7,7 +7,7 @@ AdaptiveSparkPlan (20) +- ShuffleQueryStage (8), Statistics(X) +- ColumnarExchange (7) +- VeloxAppendBatches (6) - +- ^ RegularHashAggregateExecTransformer (4) + +- ^ FlushableHashAggregateExecTransformer (4) +- ^ ProjectExecTransformer (3) +- ^ NoopFilter (2) +- ^ Scan parquet (1) @@ -35,7 +35,7 @@ Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) RegularHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt index 5bccbe75ed31..71742ea423b5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt @@ -1,120 +1,121 @@ == Physical Plan == -AdaptiveSparkPlan (138) +AdaptiveSparkPlan (139) +- == Final Plan == - VeloxColumnarToRowExec (100) - +- ^ SortExecTransformer (98) - +- ^ InputIteratorTransformer (97) - +- ShuffleQueryStage (95), Statistics(X) - +- ColumnarExchange (94) - +- VeloxAppendBatches (93) - +- ^ RegularHashAggregateExecTransformer (91) - +- ^ InputIteratorTransformer (90) - +- ShuffleQueryStage (88), Statistics(X) - +- ColumnarExchange (87) - +- ^ ProjectExecTransformer (85) - +- ^ FlushableHashAggregateExecTransformer (84) - +- ^ ProjectExecTransformer (83) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) - :- ^ InputIteratorTransformer (77) - : +- ShuffleQueryStage (75), Statistics(X) - : +- ColumnarExchange (74) - : +- VeloxAppendBatches (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - : :- ^ InputIteratorTransformer (60) - : : +- ShuffleQueryStage (58), Statistics(X) - : : +- ColumnarExchange (57) - : : +- VeloxAppendBatches (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - : : :- ^ InputIteratorTransformer (43) - : : : +- ShuffleQueryStage (41), Statistics(X) - : : : +- ColumnarExchange (40) - : : : +- VeloxAppendBatches (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : :- ^ InputIteratorTransformer (26) - : : : : +- ShuffleQueryStage (24), Statistics(X) - : : : : +- ColumnarExchange (23) - : : : : +- VeloxAppendBatches (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ InputIteratorTransformer (9) - : : : : : +- ShuffleQueryStage (7), Statistics(X) - : : : : : +- ColumnarExchange (6) - : : : : : +- VeloxAppendBatches (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (35) - : : : +- ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- VeloxAppendBatches (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ NoopFilter (28) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50), Statistics(X) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ NoopFilter (45) - : : +- ^ Scan parquet (44) - : +- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67), Statistics(X) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ NoopFilter (62) - : +- ^ Scan parquet (61) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ReusedExchange (78) + VeloxColumnarToRowExec (101) + +- ^ SortExecTransformer (99) + +- ^ InputIteratorTransformer (98) + +- ShuffleQueryStage (96), Statistics(X) + +- ColumnarExchange (95) + +- VeloxAppendBatches (94) + +- ^ RegularHashAggregateExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- VeloxAppendBatches (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ReusedExchange (78) +- == Initial Plan == - Sort (137) - +- Exchange (136) - +- HashAggregate (135) - +- Exchange (134) - +- HashAggregate (133) - +- Project (132) - +- ShuffledHashJoin Inner BuildRight (131) - :- Exchange (127) - : +- Project (126) - : +- ShuffledHashJoin Inner BuildRight (125) - : :- Exchange (121) - : : +- Project (120) - : : +- ShuffledHashJoin Inner BuildRight (119) - : : :- Exchange (115) - : : : +- Project (114) - : : : +- ShuffledHashJoin Inner BuildRight (113) - : : : :- Exchange (109) - : : : : +- Project (108) - : : : : +- ShuffledHashJoin Inner BuildLeft (107) - : : : : :- Exchange (103) - : : : : : +- Filter (102) - : : : : : +- Scan parquet (101) - : : : : +- Exchange (106) - : : : : +- Filter (105) - : : : : +- Scan parquet (104) - : : : +- Exchange (112) - : : : +- Filter (111) - : : : +- Scan parquet (110) - : : +- Exchange (118) - : : +- Filter (117) - : : +- Scan parquet (116) - : +- Exchange (124) - : +- Filter (123) - : +- Scan parquet (122) - +- Exchange (130) - +- Filter (129) - +- Scan parquet (128) + Sort (138) + +- Exchange (137) + +- HashAggregate (136) + +- Exchange (135) + +- HashAggregate (134) + +- Project (133) + +- ShuffledHashJoin Inner BuildRight (132) + :- Exchange (128) + : +- Project (127) + : +- ShuffledHashJoin Inner BuildRight (126) + : :- Exchange (122) + : : +- Project (121) + : : +- ShuffledHashJoin Inner BuildRight (120) + : : :- Exchange (116) + : : : +- Project (115) + : : : +- ShuffledHashJoin Inner BuildRight (114) + : : : :- Exchange (110) + : : : : +- Project (109) + : : : : +- ShuffledHashJoin Inner BuildLeft (108) + : : : : :- Exchange (104) + : : : : : +- Filter (103) + : : : : : +- Scan parquet (102) + : : : : +- Exchange (107) + : : : : +- Filter (106) + : : : : +- Scan parquet (105) + : : : +- Exchange (113) + : : : +- Filter (112) + : : : +- Scan parquet (111) + : : +- Exchange (119) + : : +- Filter (118) + : : +- Scan parquet (117) + : +- Exchange (125) + : +- Filter (124) + : +- Scan parquet (123) + +- Exchange (131) + +- Filter (130) + +- Scan parquet (129) (1) Scan parquet @@ -463,237 +464,241 @@ Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(87) ColumnarExchange +(87) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(88) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(88) ShuffleQueryStage +(89) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(89) InputAdapter +(90) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(90) InputIteratorTransformer +(91) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(91) RegularHashAggregateExecTransformer +(92) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(92) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(93) VeloxAppendBatches +(94) VeloxAppendBatches Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(94) ColumnarExchange +(95) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(95) ShuffleQueryStage +(96) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(96) InputAdapter +(97) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(97) InputIteratorTransformer +(98) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(98) SortExecTransformer +(99) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(99) WholeStageCodegenTransformer (X) +(100) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(100) VeloxColumnarToRowExec +(101) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(101) Scan parquet +(102) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(102) Filter +(103) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(103) Exchange +(104) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) Scan parquet +(105) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(105) Filter +(106) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(106) Exchange +(107) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(108) Project +(109) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(109) Exchange +(110) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(112) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(114) Project +(115) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(115) Exchange +(116) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) Scan parquet +(117) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(117) Filter +(118) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(118) Exchange +(119) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(120) Project +(121) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(121) Exchange +(122) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(123) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(123) Filter +(124) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(124) Exchange +(125) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(126) Project +(127) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(127) Exchange +(128) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(129) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(129) Filter +(130) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(130) Exchange +(131) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(132) Project +(133) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(133) HashAggregate +(134) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(134) Exchange +(135) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(135) HashAggregate +(136) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(136) Exchange +(137) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) Sort +(138) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(138) AdaptiveSparkPlan +(139) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt index 553bb62a242e..319e6c9f1b21 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt @@ -1,165 +1,166 @@ == Physical Plan == -AdaptiveSparkPlan (192) +AdaptiveSparkPlan (193) +- == Final Plan == - VeloxColumnarToRowExec (140) - +- ^ SortExecTransformer (138) - +- ^ InputIteratorTransformer (137) - +- ShuffleQueryStage (135), Statistics(X) - +- ColumnarExchange (134) - +- VeloxAppendBatches (133) - +- ^ ProjectExecTransformer (131) - +- ^ RegularHashAggregateExecTransformer (130) - +- ^ InputIteratorTransformer (129) - +- ShuffleQueryStage (127), Statistics(X) - +- ColumnarExchange (126) - +- ^ ProjectExecTransformer (124) - +- ^ FlushableHashAggregateExecTransformer (123) - +- ^ ProjectExecTransformer (122) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) - :- ^ InputIteratorTransformer (111) - : +- ShuffleQueryStage (109), Statistics(X) - : +- ColumnarExchange (108) - : +- VeloxAppendBatches (107) - : +- ^ ProjectExecTransformer (105) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) - : :- ^ InputIteratorTransformer (94) - : : +- ShuffleQueryStage (92), Statistics(X) - : : +- ColumnarExchange (91) - : : +- VeloxAppendBatches (90) - : : +- ^ ProjectExecTransformer (88) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - : : :- ^ InputIteratorTransformer (77) - : : : +- ShuffleQueryStage (75), Statistics(X) - : : : +- ColumnarExchange (74) - : : : +- VeloxAppendBatches (73) - : : : +- ^ ProjectExecTransformer (71) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) - : : : :- ^ InputIteratorTransformer (60) - : : : : +- ShuffleQueryStage (58), Statistics(X) - : : : : +- ColumnarExchange (57) - : : : : +- VeloxAppendBatches (56) - : : : : +- ^ ProjectExecTransformer (54) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) - : : : : :- ^ InputIteratorTransformer (43) - : : : : : +- ShuffleQueryStage (41), Statistics(X) - : : : : : +- ColumnarExchange (40) - : : : : : +- VeloxAppendBatches (39) - : : : : : +- ^ ProjectExecTransformer (37) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : : : :- ^ InputIteratorTransformer (26) - : : : : : : +- ShuffleQueryStage (24), Statistics(X) - : : : : : : +- ColumnarExchange (23) - : : : : : : +- VeloxAppendBatches (22) - : : : : : : +- ^ ProjectExecTransformer (20) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : : : :- ^ InputIteratorTransformer (9) - : : : : : : : +- ShuffleQueryStage (7), Statistics(X) - : : : : : : : +- ColumnarExchange (6) - : : : : : : : +- VeloxAppendBatches (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (18) - : : : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : : : +- ColumnarExchange (15) - : : : : : : +- VeloxAppendBatches (14) - : : : : : : +- ^ ProjectExecTransformer (12) - : : : : : : +- ^ NoopFilter (11) - : : : : : : +- ^ Scan parquet (10) - : : : : : +- ^ InputIteratorTransformer (35) - : : : : : +- ShuffleQueryStage (33), Statistics(X) - : : : : : +- ColumnarExchange (32) - : : : : : +- VeloxAppendBatches (31) - : : : : : +- ^ ProjectExecTransformer (29) - : : : : : +- ^ NoopFilter (28) - : : : : : +- ^ Scan parquet (27) - : : : : +- ^ InputIteratorTransformer (52) - : : : : +- ShuffleQueryStage (50), Statistics(X) - : : : : +- ColumnarExchange (49) - : : : : +- VeloxAppendBatches (48) - : : : : +- ^ ProjectExecTransformer (46) - : : : : +- ^ NoopFilter (45) - : : : : +- ^ Scan parquet (44) - : : : +- ^ InputIteratorTransformer (69) - : : : +- ShuffleQueryStage (67), Statistics(X) - : : : +- ColumnarExchange (66) - : : : +- VeloxAppendBatches (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ NoopFilter (62) - : : : +- ^ Scan parquet (61) - : : +- ^ InputIteratorTransformer (86) - : : +- ShuffleQueryStage (84), Statistics(X) - : : +- ColumnarExchange (83) - : : +- VeloxAppendBatches (82) - : : +- ^ ProjectExecTransformer (80) - : : +- ^ NoopFilter (79) - : : +- ^ Scan parquet (78) - : +- ^ InputIteratorTransformer (103) - : +- ShuffleQueryStage (101), Statistics(X) - : +- ColumnarExchange (100) - : +- VeloxAppendBatches (99) - : +- ^ ProjectExecTransformer (97) - : +- ^ NoopFilter (96) - : +- ^ Scan parquet (95) - +- ^ InputIteratorTransformer (120) - +- ShuffleQueryStage (118), Statistics(X) - +- ColumnarExchange (117) - +- VeloxAppendBatches (116) - +- ^ ProjectExecTransformer (114) - +- ^ NoopFilter (113) - +- ^ Scan parquet (112) + VeloxColumnarToRowExec (141) + +- ^ SortExecTransformer (139) + +- ^ InputIteratorTransformer (138) + +- ShuffleQueryStage (136), Statistics(X) + +- ColumnarExchange (135) + +- VeloxAppendBatches (134) + +- ^ ProjectExecTransformer (132) + +- ^ RegularHashAggregateExecTransformer (131) + +- ^ InputIteratorTransformer (130) + +- ShuffleQueryStage (128), Statistics(X) + +- ColumnarExchange (127) + +- VeloxAppendBatches (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109), Statistics(X) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92), Statistics(X) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75), Statistics(X) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58), Statistics(X) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41), Statistics(X) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ NoopFilter (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33), Statistics(X) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ NoopFilter (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50), Statistics(X) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ NoopFilter (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67), Statistics(X) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ NoopFilter (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ NoopFilter (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ NoopFilter (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118), Statistics(X) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ NoopFilter (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (191) - +- Exchange (190) - +- HashAggregate (189) - +- Exchange (188) - +- HashAggregate (187) - +- Project (186) - +- ShuffledHashJoin Inner BuildRight (185) - :- Exchange (180) - : +- Project (179) - : +- ShuffledHashJoin Inner BuildRight (178) - : :- Exchange (174) - : : +- Project (173) - : : +- ShuffledHashJoin Inner BuildRight (172) - : : :- Exchange (168) - : : : +- Project (167) - : : : +- ShuffledHashJoin Inner BuildRight (166) - : : : :- Exchange (162) - : : : : +- Project (161) - : : : : +- ShuffledHashJoin Inner BuildRight (160) - : : : : :- Exchange (156) - : : : : : +- Project (155) - : : : : : +- ShuffledHashJoin Inner BuildRight (154) - : : : : : :- Exchange (150) - : : : : : : +- Project (149) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (148) - : : : : : : :- Exchange (144) - : : : : : : : +- Project (143) - : : : : : : : +- Filter (142) - : : : : : : : +- Scan parquet (141) - : : : : : : +- Exchange (147) - : : : : : : +- Filter (146) - : : : : : : +- Scan parquet (145) - : : : : : +- Exchange (153) - : : : : : +- Filter (152) - : : : : : +- Scan parquet (151) - : : : : +- Exchange (159) - : : : : +- Filter (158) - : : : : +- Scan parquet (157) - : : : +- Exchange (165) - : : : +- Filter (164) - : : : +- Scan parquet (163) - : : +- Exchange (171) - : : +- Filter (170) - : : +- Scan parquet (169) - : +- Exchange (177) - : +- Filter (176) - : +- Scan parquet (175) - +- Exchange (184) - +- Project (183) - +- Filter (182) - +- Scan parquet (181) + Sort (192) + +- Exchange (191) + +- HashAggregate (190) + +- Exchange (189) + +- HashAggregate (188) + +- Project (187) + +- ShuffledHashJoin Inner BuildRight (186) + :- Exchange (181) + : +- Project (180) + : +- ShuffledHashJoin Inner BuildRight (179) + : :- Exchange (175) + : : +- Project (174) + : : +- ShuffledHashJoin Inner BuildRight (173) + : : :- Exchange (169) + : : : +- Project (168) + : : : +- ShuffledHashJoin Inner BuildRight (167) + : : : :- Exchange (163) + : : : : +- Project (162) + : : : : +- ShuffledHashJoin Inner BuildRight (161) + : : : : :- Exchange (157) + : : : : : +- Project (156) + : : : : : +- ShuffledHashJoin Inner BuildRight (155) + : : : : : :- Exchange (151) + : : : : : : +- Project (150) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (149) + : : : : : : :- Exchange (145) + : : : : : : : +- Project (144) + : : : : : : : +- Filter (143) + : : : : : : : +- Scan parquet (142) + : : : : : : +- Exchange (148) + : : : : : : +- Filter (147) + : : : : : : +- Scan parquet (146) + : : : : : +- Exchange (154) + : : : : : +- Filter (153) + : : : : : +- Scan parquet (152) + : : : : +- Exchange (160) + : : : : +- Filter (159) + : : : : +- Scan parquet (158) + : : : +- Exchange (166) + : : : +- Filter (165) + : : : +- Scan parquet (164) + : : +- Exchange (172) + : : +- Filter (171) + : : +- Scan parquet (170) + : +- Exchange (178) + : +- Filter (177) + : +- Scan parquet (176) + +- Exchange (185) + +- Project (184) + +- Filter (183) + +- Scan parquet (182) (1) Scan parquet @@ -668,305 +669,309 @@ Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(126) ColumnarExchange +(126) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(127) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(127) ShuffleQueryStage +(128) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(128) InputAdapter +(129) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(129) InputIteratorTransformer +(130) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(130) RegularHashAggregateExecTransformer +(131) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(131) ProjectExecTransformer +(132) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(132) WholeStageCodegenTransformer (X) +(133) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(133) VeloxAppendBatches +(134) VeloxAppendBatches Input [2]: [o_year#X, mkt_share#X] Arguments: X -(134) ColumnarExchange +(135) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(135) ShuffleQueryStage +(136) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(136) InputAdapter +(137) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(137) InputIteratorTransformer +(138) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(138) SortExecTransformer +(139) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(139) WholeStageCodegenTransformer (X) +(140) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(140) VeloxColumnarToRowExec +(141) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(141) Scan parquet +(142) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(142) Filter +(143) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(143) Project +(144) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(144) Exchange +(145) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) Scan parquet +(146) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(146) Filter +(147) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(147) Exchange +(148) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) ShuffledHashJoin +(149) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(149) Project +(150) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(150) Exchange +(151) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) Scan parquet +(152) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(152) Filter +(153) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(153) Exchange +(154) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) ShuffledHashJoin +(155) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(155) Project +(156) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(156) Exchange +(157) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) Scan parquet +(158) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(158) Filter +(159) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(159) Exchange +(160) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) ShuffledHashJoin +(161) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(161) Project +(162) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(162) Exchange +(163) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) Scan parquet +(164) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(164) Filter +(165) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(165) Exchange +(166) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) ShuffledHashJoin +(167) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(167) Project +(168) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(168) Exchange +(169) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(169) Scan parquet +(170) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(170) Filter +(171) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(171) Exchange +(172) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(172) ShuffledHashJoin +(173) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(173) Project +(174) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(174) Exchange +(175) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(175) Scan parquet +(176) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(176) Filter +(177) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(177) Exchange +(178) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(178) ShuffledHashJoin +(179) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(179) Project +(180) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(180) Exchange +(181) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(181) Scan parquet +(182) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(182) Filter +(183) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(183) Project +(184) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(184) Exchange +(185) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(185) ShuffledHashJoin +(186) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(186) Project +(187) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(187) HashAggregate +(188) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(188) Exchange +(189) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(189) HashAggregate +(190) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(190) Exchange +(191) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(191) Sort +(192) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(192) AdaptiveSparkPlan +(193) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt index f894a33ab6e7..40dee1752399 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt @@ -1,125 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (144) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (105) - +- ^ SortExecTransformer (103) - +- ^ InputIteratorTransformer (102) - +- ShuffleQueryStage (100), Statistics(X) - +- ColumnarExchange (99) - +- VeloxAppendBatches (98) - +- ^ RegularHashAggregateExecTransformer (96) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ColumnarExchange (92) - +- ^ ProjectExecTransformer (90) - +- ^ FlushableHashAggregateExecTransformer (89) - +- ^ ProjectExecTransformer (88) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - :- ^ InputIteratorTransformer (77) - : +- ShuffleQueryStage (75), Statistics(X) - : +- ColumnarExchange (74) - : +- VeloxAppendBatches (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) - : :- ^ InputIteratorTransformer (60) - : : +- ShuffleQueryStage (58), Statistics(X) - : : +- ColumnarExchange (57) - : : +- VeloxAppendBatches (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) - : : :- ^ InputIteratorTransformer (43) - : : : +- ShuffleQueryStage (41), Statistics(X) - : : : +- ColumnarExchange (40) - : : : +- VeloxAppendBatches (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : :- ^ InputIteratorTransformer (26) - : : : : +- ShuffleQueryStage (24), Statistics(X) - : : : : +- ColumnarExchange (23) - : : : : +- VeloxAppendBatches (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ InputIteratorTransformer (9) - : : : : : +- ShuffleQueryStage (7), Statistics(X) - : : : : : +- ColumnarExchange (6) - : : : : : +- VeloxAppendBatches (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (35) - : : : +- ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- VeloxAppendBatches (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ NoopFilter (28) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50), Statistics(X) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ NoopFilter (45) - : : +- ^ Scan parquet (44) - : +- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67), Statistics(X) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ NoopFilter (62) - : +- ^ Scan parquet (61) - +- ^ InputIteratorTransformer (86) - +- ShuffleQueryStage (84), Statistics(X) - +- ColumnarExchange (83) - +- VeloxAppendBatches (82) - +- ^ ProjectExecTransformer (80) - +- ^ NoopFilter (79) - +- ^ Scan parquet (78) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94), Statistics(X) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (143) - +- Exchange (142) - +- HashAggregate (141) - +- Exchange (140) - +- HashAggregate (139) - +- Project (138) - +- ShuffledHashJoin Inner BuildRight (137) - :- Exchange (133) - : +- Project (132) - : +- ShuffledHashJoin Inner BuildRight (131) - : :- Exchange (127) - : : +- Project (126) - : : +- ShuffledHashJoin Inner BuildRight (125) - : : :- Exchange (121) - : : : +- Project (120) - : : : +- ShuffledHashJoin Inner BuildRight (119) - : : : :- Exchange (115) - : : : : +- Project (114) - : : : : +- ShuffledHashJoin Inner BuildLeft (113) - : : : : :- Exchange (109) - : : : : : +- Project (108) - : : : : : +- Filter (107) - : : : : : +- Scan parquet (106) - : : : : +- Exchange (112) - : : : : +- Filter (111) - : : : : +- Scan parquet (110) - : : : +- Exchange (118) - : : : +- Filter (117) - : : : +- Scan parquet (116) - : : +- Exchange (124) - : : +- Filter (123) - : : +- Scan parquet (122) - : +- Exchange (130) - : +- Filter (129) - : +- Scan parquet (128) - +- Exchange (136) - +- Filter (135) - +- Scan parquet (134) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (110) + : : : : : +- Project (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Filter (112) + : : : : +- Scan parquet (111) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -492,241 +493,245 @@ Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(92) ColumnarExchange +(92) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(93) ShuffleQueryStage +(94) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(94) InputAdapter +(95) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(95) InputIteratorTransformer +(96) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(96) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(97) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(98) VeloxAppendBatches +(99) VeloxAppendBatches Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(99) ColumnarExchange +(100) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(100) ShuffleQueryStage +(101) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(101) InputAdapter +(102) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(102) InputIteratorTransformer +(103) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(103) SortExecTransformer +(104) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(104) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(105) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(106) Scan parquet +(107) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(107) Filter +(108) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(108) Project +(109) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(109) Exchange +(110) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(111) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(112) Exchange +(113) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(114) Project +(115) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(115) Exchange +(116) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) Scan parquet +(117) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(117) Filter +(118) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(118) Exchange +(119) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(120) Project +(121) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(121) Exchange +(122) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(123) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(123) Filter +(124) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(124) Exchange +(125) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(126) Project +(127) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(127) Exchange +(128) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(129) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(129) Filter +(130) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(130) Exchange +(131) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(132) Project +(133) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(133) Exchange +(134) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(134) Scan parquet +(135) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(135) Filter +(136) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(136) Exchange +(137) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(138) Project +(139) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(139) HashAggregate +(140) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(140) Exchange +(141) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(141) HashAggregate +(142) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(142) Exchange +(143) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(143) Sort +(144) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(144) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt index 589019c4cdbf..545f2e7e086d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (29) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (28) - +- Exchange (27) - +- HashAggregate (26) - +- Exchange (25) - +- HashAggregate (24) - +- Project (23) - +- Filter (22) - +- Scan parquet (21) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -57,101 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) VeloxAppendBatches +(14) VeloxAppendBatches Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) ColumnarExchange +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(18) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(20) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(21) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(22) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(23) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(24) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(25) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(27) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(29) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt index e26d0ed210e2..94e1100ea37d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt @@ -1,84 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (66) - +- TakeOrderedAndProjectExecTransformer (65) - +- ^ ProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ ProjectExecTransformer (56) - +- ^ FlushableHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - :- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41), Statistics(X) - : +- ColumnarExchange (40) - : +- VeloxAppendBatches (39) - : +- ^ ProjectExecTransformer (37) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - : :- ^ InputIteratorTransformer (26) - : : +- ShuffleQueryStage (24), Statistics(X) - : : +- ColumnarExchange (23) - : : +- VeloxAppendBatches (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : : :- ^ InputIteratorTransformer (9) - : : : +- ShuffleQueryStage (7), Statistics(X) - : : : +- ColumnarExchange (6) - : : : +- VeloxAppendBatches (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (18) - : : +- ShuffleQueryStage (16), Statistics(X) - : : +- ColumnarExchange (15) - : : +- VeloxAppendBatches (14) - : : +- ^ ProjectExecTransformer (12) - : : +- ^ NoopFilter (11) - : : +- ^ Scan parquet (10) - : +- ^ InputIteratorTransformer (35) - : +- ShuffleQueryStage (33), Statistics(X) - : +- ColumnarExchange (32) - : +- VeloxAppendBatches (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ NoopFilter (28) - : +- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- VeloxAppendBatches (48) - +- ^ ProjectExecTransformer (46) - +- ^ NoopFilter (45) - +- ^ Scan parquet (44) + VeloxColumnarToRowExec (67) + +- TakeOrderedAndProjectExecTransformer (66) + +- ^ ProjectExecTransformer (64) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24), Statistics(X) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ NoopFilter (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33), Statistics(X) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ NoopFilter (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ NoopFilter (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (92) - +- HashAggregate (91) - +- Exchange (90) - +- HashAggregate (89) - +- Project (88) - +- ShuffledHashJoin Inner BuildRight (87) - :- Exchange (83) - : +- Project (82) - : +- ShuffledHashJoin Inner BuildRight (81) - : :- Exchange (76) - : : +- Project (75) - : : +- ShuffledHashJoin Inner BuildRight (74) - : : :- Exchange (69) - : : : +- Filter (68) - : : : +- Scan parquet (67) - : : +- Exchange (73) - : : +- Project (72) - : : +- Filter (71) - : : +- Scan parquet (70) - : +- Exchange (80) - : +- Project (79) - : +- Filter (78) - : +- Scan parquet (77) - +- Exchange (86) - +- Filter (85) - +- Scan parquet (84) + TakeOrderedAndProject (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- ShuffledHashJoin Inner BuildRight (88) + :- Exchange (84) + : +- Project (83) + : +- ShuffledHashJoin Inner BuildRight (82) + : :- Exchange (77) + : : +- Project (76) + : : +- ShuffledHashJoin Inner BuildRight (75) + : : :- Exchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- Exchange (74) + : : +- Project (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (81) + : +- Project (80) + : +- Filter (79) + : +- Scan parquet (78) + +- Exchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -318,170 +319,174 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(58) ColumnarExchange +(58) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(59) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(60) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(60) InputAdapter +(61) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(61) InputIteratorTransformer +(62) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(62) RegularHashAggregateExecTransformer +(63) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(63) ProjectExecTransformer +(64) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(65) TakeOrderedAndProjectExecTransformer +(66) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(66) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) Scan parquet +(68) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(68) Filter +(69) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(69) Exchange +(70) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(70) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(72) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(73) Exchange +(74) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) ShuffledHashJoin +(75) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(75) Project +(76) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(76) Exchange +(77) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Scan parquet +(78) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(78) Filter +(79) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(79) Project +(80) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(80) Exchange +(81) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(82) Project +(83) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(83) Exchange +(84) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(86) Exchange +(87) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) ShuffledHashJoin +(88) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(88) Project +(89) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(89) HashAggregate +(90) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(90) Exchange +(91) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(92) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(92) TakeOrderedAndProject +(93) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(93) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt index a360e2ccdf4d..41d23099319f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt @@ -1,70 +1,71 @@ == Physical Plan == -AdaptiveSparkPlan (77) +AdaptiveSparkPlan (78) +- == Final Plan == - VeloxColumnarToRowExec (55) - +- ^ SortExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- VeloxAppendBatches (48) - +- ^ FilterExecTransformer (46) - +- ^ RegularHashAggregateExecTransformer (45) - +- ^ InputIteratorTransformer (44) - +- ShuffleQueryStage (42), Statistics(X) - +- ColumnarExchange (41) - +- ^ ProjectExecTransformer (39) - +- ^ FlushableHashAggregateExecTransformer (38) - +- ^ ProjectExecTransformer (37) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - :- ^ InputIteratorTransformer (26) - : +- ShuffleQueryStage (24), Statistics(X) - : +- ColumnarExchange (23) - : +- VeloxAppendBatches (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : :- ^ InputIteratorTransformer (9) - : : +- ShuffleQueryStage (7), Statistics(X) - : : +- ColumnarExchange (6) - : : +- VeloxAppendBatches (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (18) - : +- ShuffleQueryStage (16), Statistics(X) - : +- ColumnarExchange (15) - : +- VeloxAppendBatches (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ NoopFilter (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- VeloxAppendBatches (31) - +- ^ ProjectExecTransformer (29) - +- ^ NoopFilter (28) - +- ^ Scan parquet (27) + VeloxColumnarToRowExec (56) + +- ^ SortExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ShuffleQueryStage (51), Statistics(X) + +- ColumnarExchange (50) + +- VeloxAppendBatches (49) + +- ^ FilterExecTransformer (47) + +- ^ RegularHashAggregateExecTransformer (46) + +- ^ InputIteratorTransformer (45) + +- ShuffleQueryStage (43), Statistics(X) + +- ColumnarExchange (42) + +- VeloxAppendBatches (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ NoopFilter (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (76) - +- Exchange (75) - +- Filter (74) - +- HashAggregate (73) - +- Exchange (72) - +- HashAggregate (71) - +- Project (70) - +- ShuffledHashJoin Inner BuildRight (69) - :- Exchange (64) - : +- Project (63) - : +- ShuffledHashJoin Inner BuildRight (62) - : :- Exchange (58) - : : +- Filter (57) - : : +- Scan parquet (56) - : +- Exchange (61) - : +- Filter (60) - : +- Scan parquet (59) - +- Exchange (68) - +- Project (67) - +- Filter (66) - +- Scan parquet (65) + Sort (77) + +- Exchange (76) + +- Filter (75) + +- HashAggregate (74) + +- Exchange (73) + +- HashAggregate (72) + +- Project (71) + +- ShuffledHashJoin Inner BuildRight (70) + :- Exchange (65) + : +- Project (64) + : +- ShuffledHashJoin Inner BuildRight (63) + : :- Exchange (59) + : : +- Filter (58) + : : +- Scan parquet (57) + : +- Exchange (62) + : +- Filter (61) + : +- Scan parquet (60) + +- Exchange (69) + +- Project (68) + +- Filter (67) + +- Scan parquet (66) (1) Scan parquet @@ -235,438 +236,442 @@ Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(41) ColumnarExchange +(41) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(42) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(42) ShuffleQueryStage +(43) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(43) InputAdapter +(44) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(44) InputIteratorTransformer +(45) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(45) RegularHashAggregateExecTransformer +(46) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(46) FilterExecTransformer +(47) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(47) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(48) VeloxAppendBatches +(49) VeloxAppendBatches Input [2]: [ps_partkey#X, value#X] Arguments: X -(49) ColumnarExchange +(50) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(51) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(51) InputAdapter +(52) InputAdapter Input [2]: [ps_partkey#X, value#X] -(52) InputIteratorTransformer +(53) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(53) SortExecTransformer +(54) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(54) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(55) VeloxColumnarToRowExec +(56) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(56) Scan parquet +(57) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(57) Filter +(58) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(58) Exchange +(59) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) Scan parquet +(60) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(60) Filter +(61) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(61) Exchange +(62) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) ShuffledHashJoin +(63) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(63) Project +(64) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(64) Exchange +(65) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) Scan parquet +(66) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(67) Project +(68) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(68) Exchange +(69) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(69) ShuffledHashJoin +(70) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(70) Project +(71) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(71) HashAggregate +(72) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(72) Exchange +(73) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(73) HashAggregate +(74) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(74) Filter +(75) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(75) Exchange +(76) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) Sort +(77) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(77) AdaptiveSparkPlan +(78) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (127) +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (109) - +- ^ ProjectExecTransformer (107) - +- ^ RegularHashAggregateExecTransformer (106) - +- ^ RegularHashAggregateExecTransformer (105) - +- ^ ProjectExecTransformer (104) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (103) - :- ^ InputIteratorTransformer (98) - : +- ShuffleQueryStage (96), Statistics(X) - : +- ColumnarExchange (95) - : +- VeloxAppendBatches (94) - : +- ^ ProjectExecTransformer (92) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (91) - : :- ^ InputIteratorTransformer (86) - : : +- ShuffleQueryStage (84), Statistics(X) - : : +- ColumnarExchange (83) - : : +- VeloxAppendBatches (82) - : : +- ^ ProjectExecTransformer (80) - : : +- ^ NoopFilter (79) - : : +- ^ Scan parquet (78) - : +- ^ InputIteratorTransformer (90) - : +- ShuffleQueryStage (88), Statistics(X) - : +- ReusedExchange (87) - +- ^ InputIteratorTransformer (102) - +- ShuffleQueryStage (100), Statistics(X) - +- ReusedExchange (99) + VeloxColumnarToRowExec (110) + +- ^ ProjectExecTransformer (108) + +- ^ RegularHashAggregateExecTransformer (107) + +- ^ RegularHashAggregateExecTransformer (106) + +- ^ ProjectExecTransformer (105) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + :- ^ InputIteratorTransformer (99) + : +- ShuffleQueryStage (97), Statistics(X) + : +- ColumnarExchange (96) + : +- VeloxAppendBatches (95) + : +- ^ ProjectExecTransformer (93) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) + : :- ^ InputIteratorTransformer (87) + : : +- ShuffleQueryStage (85), Statistics(X) + : : +- ColumnarExchange (84) + : : +- VeloxAppendBatches (83) + : : +- ^ ProjectExecTransformer (81) + : : +- ^ NoopFilter (80) + : : +- ^ Scan parquet (79) + : +- ^ InputIteratorTransformer (91) + : +- ShuffleQueryStage (89), Statistics(X) + : +- ReusedExchange (88) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ReusedExchange (100) +- == Initial Plan == - HashAggregate (126) - +- HashAggregate (125) - +- Project (124) - +- ShuffledHashJoin Inner BuildRight (123) - :- Exchange (118) - : +- Project (117) - : +- ShuffledHashJoin Inner BuildRight (116) - : :- Exchange (112) - : : +- Filter (111) - : : +- Scan parquet (110) - : +- Exchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- Exchange (122) - +- Project (121) - +- Filter (120) - +- Scan parquet (119) - - -(78) Scan parquet + HashAggregate (127) + +- HashAggregate (126) + +- Project (125) + +- ShuffledHashJoin Inner BuildRight (124) + :- Exchange (119) + : +- Project (118) + : +- ShuffledHashJoin Inner BuildRight (117) + : :- Exchange (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Exchange (116) + : +- Filter (115) + : +- Scan parquet (114) + +- Exchange (123) + +- Project (122) + +- Filter (121) + +- Scan parquet (120) + + +(79) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(79) NoopFilter +(80) NoopFilter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(80) ProjectExecTransformer +(81) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(81) WholeStageCodegenTransformer (X) +(82) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(82) VeloxAppendBatches +(83) VeloxAppendBatches Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(83) ColumnarExchange +(84) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(84) ShuffleQueryStage +(85) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(85) InputAdapter +(86) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(86) InputIteratorTransformer +(87) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(87) ReusedExchange [Reuses operator id: 15] +(88) ReusedExchange [Reuses operator id: 15] Output [2]: [s_suppkey#X, s_nationkey#X] -(88) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(89) InputAdapter +(90) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(90) InputIteratorTransformer +(91) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(91) ShuffledHashJoinExecTransformer +(92) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(92) ProjectExecTransformer +(93) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(93) WholeStageCodegenTransformer (X) +(94) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(94) VeloxAppendBatches +(95) VeloxAppendBatches Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(95) ColumnarExchange +(96) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(97) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(97) InputAdapter +(98) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(98) InputIteratorTransformer +(99) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(99) ReusedExchange [Reuses operator id: 32] +(100) ReusedExchange [Reuses operator id: 32] Output [1]: [n_nationkey#X] -(100) ShuffleQueryStage +(101) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(101) InputAdapter +(102) InputAdapter Input [1]: [n_nationkey#X] -(102) InputIteratorTransformer +(103) InputIteratorTransformer Input [1]: [n_nationkey#X] -(103) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(104) ProjectExecTransformer +(105) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(105) RegularHashAggregateExecTransformer +(106) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(106) RegularHashAggregateExecTransformer +(107) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(107) ProjectExecTransformer +(108) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(108) WholeStageCodegenTransformer (X) +(109) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(109) VeloxColumnarToRowExec +(110) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(110) Scan parquet +(111) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(112) Exchange +(113) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) Scan parquet +(114) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(114) Filter +(115) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(115) Exchange +(116) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(117) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(117) Project +(118) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(118) Exchange +(119) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) Scan parquet +(120) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(120) Filter +(121) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(121) Project +(122) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(122) Exchange +(123) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) ShuffledHashJoin +(124) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(124) Project +(125) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(125) HashAggregate +(126) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(126) HashAggregate +(127) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(127) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt index 2cd2fbbd94c0..9995164f4c49 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt @@ -1,49 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32), Statistics(X) - +- ColumnarExchange (31) - +- VeloxAppendBatches (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7), Statistics(X) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- VeloxAppendBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- Project (46) - +- ShuffledHashJoin Inner BuildLeft (45) - :- Exchange (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Exchange (44) - +- Project (43) - +- Filter (42) - +- Scan parquet (41) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin Inner BuildLeft (46) + :- Exchange (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -145,130 +146,134 @@ Input [3]: [l_shipmode#X, sum#X, sum#X] Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(24) ColumnarExchange +(24) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(25) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(28) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(29) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(30) VeloxAppendBatches +(31) VeloxAppendBatches Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(31) ColumnarExchange +(32) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) SortExecTransformer +(36) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(37) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(38) Scan parquet +(39) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(40) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(42) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(42) Filter +(43) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(43) Project +(44) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(44) Exchange +(45) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(46) Project +(47) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(47) HashAggregate +(48) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(48) Exchange +(49) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(50) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(50) Exchange +(51) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(52) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(52) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt index 7e296f215aa1..53801198fb49 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (55) +AdaptiveSparkPlan (56) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- ^ SortExecTransformer (37) - +- ^ InputIteratorTransformer (36) - +- ShuffleQueryStage (34), Statistics(X) - +- ColumnarExchange (33) - +- VeloxAppendBatches (32) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- VeloxAppendBatches (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (54) - +- Exchange (53) - +- HashAggregate (52) - +- Exchange (51) - +- HashAggregate (50) - +- HashAggregate (49) - +- HashAggregate (48) - +- Project (47) - +- ShuffledHashJoin LeftOuter BuildRight (46) - :- Exchange (41) - : +- Scan parquet (40) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftOuter BuildRight (47) + :- Exchange (42) + : +- Scan parquet (41) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -161,139 +162,143 @@ Input [2]: [c_count#X, count#X] Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [2]: [c_count#X, count#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(32) VeloxAppendBatches +(33) VeloxAppendBatches Input [2]: [c_count#X, custdist#X] Arguments: X -(33) ColumnarExchange +(34) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(34) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(35) InputAdapter +(36) InputAdapter Input [2]: [c_count#X, custdist#X] -(36) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(37) SortExecTransformer +(38) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(38) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(39) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(40) Scan parquet +(41) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(41) Exchange +(42) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Scan parquet +(43) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(43) Filter +(44) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(44) Project +(45) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(45) Exchange +(46) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(47) Project +(48) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(48) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(49) HashAggregate +(50) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(50) HashAggregate +(51) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(51) Exchange +(52) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) HashAggregate +(53) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(53) Exchange +(54) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Sort +(55) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(55) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt index 3fa54d1d647b..a177fe4bcaec 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt @@ -1,13 +1,13 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- AQEShuffleRead (28) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- VeloxAppendBatches (25) - +- ^ ProjectExecTransformer (23) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (22) + VeloxColumnarToRowExec (30) + +- AQEShuffleRead (29) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (23) :- ^ InputIteratorTransformer (9) : +- ShuffleQueryStage (7), Statistics(X) : +- ColumnarExchange (6) @@ -15,31 +15,32 @@ AdaptiveSparkPlan (44) : +- ^ ProjectExecTransformer (3) : +- ^ NoopFilter (2) : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- Project (41) - +- ShuffledHashJoin Inner BuildLeft (40) - :- Exchange (32) - : +- Filter (31) - : +- Scan parquet (30) - +- Filter (39) - +- HashAggregate (38) - +- Exchange (37) - +- HashAggregate (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (44) + +- Exchange (43) + +- Project (42) + +- ShuffledHashJoin Inner BuildLeft (41) + :- Exchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Filter (40) + +- HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -109,290 +110,299 @@ Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(16) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(21) FilterExecTransformer +(22) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(22) ShuffledHashJoinExecTransformer +(23) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(23) ProjectExecTransformer +(24) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(24) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(25) VeloxAppendBatches +(26) VeloxAppendBatches Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) ColumnarExchange +(27) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(28) AQEShuffleRead +(29) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(30) Scan parquet +(31) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(32) Exchange +(33) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(33) Scan parquet +(34) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(34) Filter +(35) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(35) Project +(36) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(36) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(37) Exchange +(38) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) HashAggregate +(39) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(39) Filter +(40) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(40) ShuffledHashJoin +(41) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(41) Project +(42) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(42) Exchange +(43) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(44) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 21 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (69) +Subquery:1 Hosting operator id = 22 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (71) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ RegularHashAggregateExecTransformer (58) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ ProjectExecTransformer (56) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ InputIteratorTransformer (54) - +- ShuffleQueryStage (52), Statistics(X) - +- ColumnarExchange (51) - +- ^ ProjectExecTransformer (49) - +- ^ FlushableHashAggregateExecTransformer (48) - +- ^ ProjectExecTransformer (47) - +- ^ NoopFilter (46) - +- ^ Scan parquet (45) + VeloxColumnarToRowExec (62) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ ProjectExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ NoopFilter (47) + +- ^ Scan parquet (46) +- == Initial Plan == - HashAggregate (68) - +- HashAggregate (67) - +- HashAggregate (66) - +- Exchange (65) - +- HashAggregate (64) - +- Project (63) - +- Filter (62) - +- Scan parquet (61) + HashAggregate (70) + +- HashAggregate (69) + +- HashAggregate (68) + +- Exchange (67) + +- HashAggregate (66) + +- Project (65) + +- Filter (64) + +- Scan parquet (63) -(45) Scan parquet +(46) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(46) NoopFilter +(47) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(47) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(48) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(49) ProjectExecTransformer +(50) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(50) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(51) ColumnarExchange +(52) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(53) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(54) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(53) InputAdapter +(55) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(54) InputIteratorTransformer +(56) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(55) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(56) ProjectExecTransformer +(58) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(57) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(58) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(59) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(60) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(61) Scan parquet +(63) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(62) Filter +(64) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(63) Project +(65) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(64) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(65) Exchange +(67) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) HashAggregate +(68) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(67) HashAggregate +(69) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(68) HashAggregate +(70) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(69) AdaptiveSparkPlan +(71) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt index 70fe9d3d51e4..89a647ffce45 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt @@ -1,62 +1,64 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (45) - +- ^ SortExecTransformer (43) - +- ^ InputIteratorTransformer (42) - +- ShuffleQueryStage (40), Statistics(X) - +- ColumnarExchange (39) - +- VeloxAppendBatches (38) - +- ^ RegularHashAggregateExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ ProjectExecTransformer (30) - +- ^ FlushableHashAggregateExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7), Statistics(X) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- VeloxAppendBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (47) + +- ^ SortExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42), Statistics(X) + +- ColumnarExchange (41) + +- VeloxAppendBatches (40) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ ProjectExecTransformer (31) + +- ^ FlushableHashAggregateExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (66) - +- Exchange (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- Project (58) - +- ShuffledHashJoin Inner BuildRight (57) - :- Exchange (53) - : +- BroadcastHashJoin LeftAnti BuildRight (52) - : :- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (51) - : +- Project (50) - : +- Filter (49) - : +- Scan parquet (48) - +- Exchange (56) - +- Filter (55) - +- Scan parquet (54) + Sort (68) + +- Exchange (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- HashAggregate (63) + +- Exchange (62) + +- HashAggregate (61) + +- Project (60) + +- ShuffledHashJoin Inner BuildRight (59) + :- Exchange (55) + : +- BroadcastHashJoin LeftAnti BuildRight (54) + : :- Filter (49) + : : +- Scan parquet (48) + : +- BroadcastExchange (53) + : +- Project (52) + : +- Filter (51) + : +- Scan parquet (50) + +- Exchange (58) + +- Filter (57) + +- Scan parquet (56) (1) Scan parquet @@ -158,205 +160,213 @@ Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(24) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(28) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(29) FlushableHashAggregateExecTransformer +(30) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(30) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(32) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(34) InputAdapter +(36) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(35) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(36) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(37) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(38) VeloxAppendBatches +(40) VeloxAppendBatches Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(39) ColumnarExchange +(41) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(42) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(41) InputAdapter +(43) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(42) InputIteratorTransformer +(44) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) SortExecTransformer +(45) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(44) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(45) VeloxColumnarToRowExec +(47) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(46) Scan parquet +(48) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(47) Filter +(49) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(48) Scan parquet +(50) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(49) Filter +(51) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(50) Project +(52) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(51) BroadcastExchange +(53) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(52) BroadcastHashJoin +(54) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: LeftAnti Join condition: None -(53) Exchange +(55) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(56) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(56) Exchange +(58) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(59) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(58) Project +(60) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(59) HashAggregate +(61) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(60) Exchange +(62) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(63) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(62) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(63) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(66) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(65) Exchange +(67) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) Sort +(68) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(67) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt index b5360f940716..42fc32b0bce1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt @@ -1,12 +1,12 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- ^ ProjectExecTransformer (37) - +- ^ RegularHashAggregateExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ ProjectExecTransformer (34) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (33) + VeloxColumnarToRowExec (40) + +- ^ ProjectExecTransformer (38) + +- ^ RegularHashAggregateExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ ProjectExecTransformer (35) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (34) :- ^ ProjectExecTransformer (20) : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) : :- ^ InputIteratorTransformer (9) @@ -23,36 +23,37 @@ AdaptiveSparkPlan (59) : +- ^ ProjectExecTransformer (12) : +- ^ NoopFilter (11) : +- ^ Scan parquet (10) - +- ^ FilterExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ NoopFilter (22) - +- ^ Scan parquet (21) + +- ^ FilterExecTransformer (33) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ NoopFilter (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (58) - +- HashAggregate (57) - +- Project (56) - +- ShuffledHashJoin Inner BuildRight (55) - :- Project (48) - : +- ShuffledHashJoin Inner BuildRight (47) - : :- Exchange (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Project (45) - : +- Filter (44) - : +- Scan parquet (43) - +- Filter (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- Filter (50) - +- Scan parquet (49) + HashAggregate (59) + +- HashAggregate (58) + +- Project (57) + +- ShuffledHashJoin Inner BuildRight (56) + :- Project (49) + : +- ShuffledHashJoin Inner BuildRight (48) + : :- Exchange (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Exchange (47) + : +- Project (46) + : +- Filter (45) + : +- Scan parquet (44) + +- Filter (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- Filter (51) + +- Scan parquet (50) (1) Scan parquet @@ -165,171 +166,175 @@ Input [3]: [l_partkey#X, sum#X, count#X] Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(32) FilterExecTransformer +(33) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(33) ShuffledHashJoinExecTransformer +(34) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(34) ProjectExecTransformer +(35) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(35) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(36) RegularHashAggregateExecTransformer +(37) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(37) ProjectExecTransformer +(38) ProjectExecTransformer Output [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(38) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(39) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(40) Scan parquet +(41) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(41) Filter +(42) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(42) Exchange +(43) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Scan parquet +(44) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(45) Project +(46) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(46) Exchange +(47) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(48) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(48) Project +(49) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(49) Scan parquet +(50) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(50) Filter +(51) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(51) HashAggregate +(52) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(52) Exchange +(53) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(54) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(54) Filter +(55) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(55) ShuffledHashJoin +(56) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(56) Project +(57) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(57) HashAggregate +(58) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(58) HashAggregate +(59) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] -(59) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt index 557e4f5258cc..ca913b2d4c84 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt @@ -1,18 +1,18 @@ == Physical Plan == -AdaptiveSparkPlan (102) +AdaptiveSparkPlan (103) +- == Final Plan == - VeloxColumnarToRowExec (69) - +- TakeOrderedAndProjectExecTransformer (68) - +- ^ RegularHashAggregateExecTransformer (66) - +- ^ RegularHashAggregateExecTransformer (65) - +- ^ ProjectExecTransformer (64) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (63) - :- ^ InputIteratorTransformer (45) - : +- ShuffleQueryStage (43), Statistics(X) - : +- ColumnarExchange (42) - : +- VeloxAppendBatches (41) - : +- ^ ProjectExecTransformer (39) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (38) + VeloxColumnarToRowExec (70) + +- TakeOrderedAndProjectExecTransformer (69) + +- ^ RegularHashAggregateExecTransformer (67) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ ProjectExecTransformer (65) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (64) + :- ^ InputIteratorTransformer (46) + : +- ShuffleQueryStage (44), Statistics(X) + : +- ColumnarExchange (43) + : +- VeloxAppendBatches (42) + : +- ^ ProjectExecTransformer (40) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (39) : :- ^ InputIteratorTransformer (9) : : +- ShuffleQueryStage (7), Statistics(X) : : +- ColumnarExchange (6) @@ -20,12 +20,12 @@ AdaptiveSparkPlan (102) : : +- ^ ProjectExecTransformer (3) : : +- ^ NoopFilter (2) : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (37) - : +- ShuffleQueryStage (35), Statistics(X) - : +- ColumnarExchange (34) - : +- VeloxAppendBatches (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (30) + : +- ^ InputIteratorTransformer (38) + : +- ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- VeloxAppendBatches (34) + : +- ^ ProjectExecTransformer (32) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (31) : :- ^ InputIteratorTransformer (18) : : +- ShuffleQueryStage (16), Statistics(X) : : +- ColumnarExchange (15) @@ -33,62 +33,63 @@ AdaptiveSparkPlan (102) : : +- ^ ProjectExecTransformer (12) : : +- ^ NoopFilter (11) : : +- ^ Scan parquet (10) - : +- ^ ProjectExecTransformer (29) - : +- ^ FilterExecTransformer (28) - : +- ^ RegularHashAggregateExecTransformer (27) - : +- ^ InputIteratorTransformer (26) - : +- ShuffleQueryStage (24), Statistics(X) - : +- ColumnarExchange (23) - : +- ^ ProjectExecTransformer (21) - : +- ^ FlushableHashAggregateExecTransformer (20) - : +- ^ Scan parquet (19) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - :- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- VeloxAppendBatches (50) - : +- ^ ProjectExecTransformer (48) - : +- ^ NoopFilter (47) - : +- ^ Scan parquet (46) - +- ^ ProjectExecTransformer (61) - +- ^ FilterExecTransformer (60) - +- ^ RegularHashAggregateExecTransformer (59) - +- ^ InputIteratorTransformer (58) - +- ShuffleQueryStage (56), Statistics(X) - +- ReusedExchange (55) + : +- ^ ProjectExecTransformer (30) + : +- ^ FilterExecTransformer (29) + : +- ^ RegularHashAggregateExecTransformer (28) + : +- ^ InputIteratorTransformer (27) + : +- ShuffleQueryStage (25), Statistics(X) + : +- ColumnarExchange (24) + : +- VeloxAppendBatches (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (63) + :- ^ InputIteratorTransformer (55) + : +- ShuffleQueryStage (53), Statistics(X) + : +- ColumnarExchange (52) + : +- VeloxAppendBatches (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ NoopFilter (48) + : +- ^ Scan parquet (47) + +- ^ ProjectExecTransformer (62) + +- ^ FilterExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57), Statistics(X) + +- ReusedExchange (56) +- == Initial Plan == - TakeOrderedAndProject (101) - +- HashAggregate (100) - +- HashAggregate (99) - +- Project (98) - +- ShuffledHashJoin Inner BuildRight (97) - :- Exchange (86) - : +- Project (85) - : +- ShuffledHashJoin Inner BuildLeft (84) - : :- Exchange (72) - : : +- Filter (71) - : : +- Scan parquet (70) - : +- Exchange (83) - : +- ShuffledHashJoin LeftSemi BuildRight (82) - : :- Exchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- Project (81) - : +- Filter (80) - : +- HashAggregate (79) - : +- Exchange (78) - : +- HashAggregate (77) - : +- Scan parquet (76) - +- ShuffledHashJoin LeftSemi BuildRight (96) - :- Exchange (89) - : +- Filter (88) - : +- Scan parquet (87) - +- Project (95) - +- Filter (94) - +- HashAggregate (93) - +- Exchange (92) - +- HashAggregate (91) - +- Scan parquet (90) + TakeOrderedAndProject (102) + +- HashAggregate (101) + +- HashAggregate (100) + +- Project (99) + +- ShuffledHashJoin Inner BuildRight (98) + :- Exchange (87) + : +- Project (86) + : +- ShuffledHashJoin Inner BuildLeft (85) + : :- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (84) + : +- ShuffledHashJoin LeftSemi BuildRight (83) + : :- Exchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Project (82) + : +- Filter (81) + : +- HashAggregate (80) + : +- Exchange (79) + : +- HashAggregate (78) + : +- Scan parquet (77) + +- ShuffledHashJoin LeftSemi BuildRight (97) + :- Exchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- Project (96) + +- Filter (95) + +- HashAggregate (94) + +- Exchange (93) + +- HashAggregate (92) + +- Scan parquet (91) (1) Scan parquet @@ -186,372 +187,376 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(28) FilterExecTransformer +(29) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(29) ProjectExecTransformer +(30) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(30) ShuffledHashJoinExecTransformer +(31) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(33) VeloxAppendBatches +(34) VeloxAppendBatches Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(34) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(35) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(36) InputAdapter +(37) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(38) ShuffledHashJoinExecTransformer +(39) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(39) ProjectExecTransformer +(40) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(40) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(41) VeloxAppendBatches +(42) VeloxAppendBatches Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(42) ColumnarExchange +(43) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(43) ShuffleQueryStage +(44) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(44) InputAdapter +(45) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(45) InputIteratorTransformer +(46) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(46) Scan parquet +(47) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(47) NoopFilter +(48) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(48) ProjectExecTransformer +(49) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(49) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(50) VeloxAppendBatches +(51) VeloxAppendBatches Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: X -(51) ColumnarExchange +(52) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(53) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(53) InputAdapter +(54) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(54) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(55) ReusedExchange [Reuses operator id: 23] +(56) ReusedExchange [Reuses operator id: 24] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(56) ShuffleQueryStage +(57) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(57) InputAdapter +(58) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(58) InputIteratorTransformer +(59) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(59) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(60) FilterExecTransformer +(61) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(61) ProjectExecTransformer +(62) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(62) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(63) ShuffledHashJoinExecTransformer +(64) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(64) ProjectExecTransformer +(65) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(65) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(66) RegularHashAggregateExecTransformer +(67) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(67) WholeStageCodegenTransformer (X) +(68) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(68) TakeOrderedAndProjectExecTransformer +(69) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(69) VeloxColumnarToRowExec +(70) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(70) Scan parquet +(71) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(72) Exchange +(73) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(73) Scan parquet +(74) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(75) Exchange +(76) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) Scan parquet +(77) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(77) HashAggregate +(78) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(78) Exchange +(79) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) HashAggregate +(80) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(80) Filter +(81) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(81) Project +(82) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(82) ShuffledHashJoin +(83) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(83) Exchange +(84) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) ShuffledHashJoin +(85) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(85) Project +(86) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(86) Exchange +(87) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Scan parquet +(88) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(88) Filter +(89) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(89) Exchange +(90) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(91) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(91) HashAggregate +(92) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(92) Exchange +(93) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) HashAggregate +(94) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(94) Filter +(95) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(95) Project +(96) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(96) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(97) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(98) Project +(99) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(99) HashAggregate +(100) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(100) HashAggregate +(101) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(101) TakeOrderedAndProject +(102) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(102) AdaptiveSparkPlan +(103) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt index f57dfa3e6f73..317740080d7b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt @@ -1,112 +1,113 @@ == Physical Plan == -AdaptiveSparkPlan (127) +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ RegularHashAggregateExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ ProjectExecTransformer (82) - +- ^ FlushableHashAggregateExecTransformer (81) - +- ^ ProjectExecTransformer (80) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) - :- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67), Statistics(X) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50), Statistics(X) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) - : : :- ^ InputIteratorTransformer (9) - : : : +- ShuffleQueryStage (7), Statistics(X) - : : : +- ColumnarExchange (6) - : : : +- VeloxAppendBatches (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (44) - : : +- ShuffleQueryStage (42), Statistics(X) - : : +- ColumnarExchange (41) - : : +- VeloxAppendBatches (40) - : : +- ^ ProjectExecTransformer (38) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) - : : : :- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (26) - : : : +- ShuffleQueryStage (24), Statistics(X) - : : : +- ColumnarExchange (23) - : : : +- VeloxAppendBatches (22) - : : : +- ^ ProjectExecTransformer (20) - : : : +- ^ Scan parquet (19) - : : +- ^ InputIteratorTransformer (36) - : : +- ShuffleQueryStage (34), Statistics(X) - : : +- ColumnarExchange (33) - : : +- VeloxAppendBatches (32) - : : +- ^ ProjectExecTransformer (30) - : : +- ^ NoopFilter (29) - : : +- ^ Scan parquet (28) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- VeloxAppendBatches (57) - : +- ^ ProjectExecTransformer (55) - : +- ^ NoopFilter (54) - : +- ^ Scan parquet (53) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- VeloxAppendBatches (74) - +- ^ ProjectExecTransformer (72) - +- ^ NoopFilter (71) - +- ^ Scan parquet (70) + VeloxColumnarToRowExec (91) + +- ^ RegularHashAggregateExecTransformer (89) + +- ^ InputIteratorTransformer (88) + +- ShuffleQueryStage (86), Statistics(X) + +- ColumnarExchange (85) + +- VeloxAppendBatches (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24), Statistics(X) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34), Statistics(X) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ NoopFilter (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ NoopFilter (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ NoopFilter (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (116) - : +- Project (115) - : +- ShuffledHashJoin Inner BuildRight (114) - : :- Exchange (109) - : : +- Project (108) - : : +- ShuffledHashJoin Inner BuildLeft (107) - : : :- Exchange (93) - : : : +- Filter (92) - : : : +- Scan parquet (91) - : : +- Exchange (106) - : : +- ShuffledHashJoin LeftAnti BuildRight (105) - : : :- ShuffledHashJoin LeftSemi BuildRight (100) - : : : :- Exchange (97) - : : : : +- Project (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (99) - : : : +- Scan parquet (98) - : : +- Exchange (104) - : : +- Project (103) - : : +- Filter (102) - : : +- Scan parquet (101) - : +- Exchange (113) - : +- Project (112) - : +- Filter (111) - : +- Scan parquet (110) - +- Exchange (120) - +- Project (119) - +- Filter (118) - +- Scan parquet (117) + TakeOrderedAndProject (127) + +- HashAggregate (126) + +- Exchange (125) + +- HashAggregate (124) + +- Project (123) + +- ShuffledHashJoin Inner BuildRight (122) + :- Exchange (117) + : +- Project (116) + : +- ShuffledHashJoin Inner BuildRight (115) + : :- Exchange (110) + : : +- Project (109) + : : +- ShuffledHashJoin Inner BuildLeft (108) + : : :- Exchange (94) + : : : +- Filter (93) + : : : +- Scan parquet (92) + : : +- Exchange (107) + : : +- ShuffledHashJoin LeftAnti BuildRight (106) + : : :- ShuffledHashJoin LeftSemi BuildRight (101) + : : : :- Exchange (98) + : : : : +- Project (97) + : : : : +- Filter (96) + : : : : +- Scan parquet (95) + : : : +- Exchange (100) + : : : +- Scan parquet (99) + : : +- Exchange (105) + : : +- Project (104) + : : +- Filter (103) + : : +- Scan parquet (102) + : +- Exchange (114) + : +- Project (113) + : +- Filter (112) + : +- Scan parquet (111) + +- Exchange (121) + +- Project (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -453,211 +454,215 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(84) ColumnarExchange +(84) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(85) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(86) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(86) InputAdapter +(87) InputAdapter Input [2]: [s_name#X, count#X] -(87) InputIteratorTransformer +(88) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(88) RegularHashAggregateExecTransformer +(89) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(89) WholeStageCodegenTransformer (X) +(90) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(90) VeloxColumnarToRowExec +(91) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(91) Scan parquet +(92) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(93) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(94) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(95) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(96) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Project +(97) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(97) Exchange +(98) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(99) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(99) Exchange +(100) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) ShuffledHashJoin +(101) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(101) Scan parquet +(102) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(102) Filter +(103) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(103) Project +(104) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(104) Exchange +(105) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) ShuffledHashJoin +(106) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(106) Exchange +(107) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(108) Project +(109) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(109) Exchange +(110) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(112) Project +(113) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(113) Exchange +(114) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(115) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(115) Project +(116) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(116) Exchange +(117) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(118) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(118) Filter +(119) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(119) Project +(120) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(120) Exchange +(121) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(122) Project +(123) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(123) HashAggregate +(124) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(124) Exchange +(125) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(126) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(126) TakeOrderedAndProject +(127) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(127) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt index 1e1b477c5c30..0d779c9160cf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt @@ -1,46 +1,47 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (50) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- VeloxAppendBatches (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7), Statistics(X) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ ProjectExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin LeftAnti BuildRight (42) - :- Exchange (39) - : +- Filter (38) - : +- Scan parquet (37) - +- Exchange (41) - +- Scan parquet (40) + Sort (49) + +- Exchange (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftAnti BuildRight (43) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (42) + +- Scan parquet (41) (1) Scan parquet @@ -137,235 +138,244 @@ Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(29) VeloxAppendBatches +(30) VeloxAppendBatches Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(30) ColumnarExchange +(31) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(32) InputAdapter +(33) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(33) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) SortExecTransformer +(35) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(35) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(36) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(37) Scan parquet +(38) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(38) Filter +(39) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(39) Exchange +(40) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) Scan parquet +(41) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(41) Exchange +(42) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(43) Project +(44) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(45) Exchange +(46) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(47) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(47) Exchange +(48) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(49) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(50) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (68) +AdaptiveSparkPlan (70) +- == Final Plan == - VeloxColumnarToRowExec (61) - +- ^ RegularHashAggregateExecTransformer (59) - +- ^ InputIteratorTransformer (58) - +- ShuffleQueryStage (56), Statistics(X) - +- ColumnarExchange (55) - +- ^ FlushableHashAggregateExecTransformer (53) - +- ^ ProjectExecTransformer (52) - +- ^ NoopFilter (51) - +- ^ Scan parquet (50) + VeloxColumnarToRowExec (63) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ InputIteratorTransformer (60) + +- ShuffleQueryStage (58), Statistics(X) + +- ColumnarExchange (57) + +- VeloxAppendBatches (56) + +- ^ FlushableHashAggregateExecTransformer (54) + +- ^ ProjectExecTransformer (53) + +- ^ NoopFilter (52) + +- ^ Scan parquet (51) +- == Initial Plan == - HashAggregate (67) - +- Exchange (66) - +- HashAggregate (65) - +- Project (64) - +- Filter (63) - +- Scan parquet (62) + HashAggregate (69) + +- Exchange (68) + +- HashAggregate (67) + +- Project (66) + +- Filter (65) + +- Scan parquet (64) -(50) Scan parquet +(51) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(51) NoopFilter +(52) NoopFilter Input [2]: [c_phone#X, c_acctbal#X] Arguments: [c_phone#X, c_acctbal#X] -(52) ProjectExecTransformer +(53) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(53) FlushableHashAggregateExecTransformer +(54) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(54) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(55) ColumnarExchange +(56) VeloxAppendBatches +Input [2]: [sum#X, count#X] +Arguments: X + +(57) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(56) ShuffleQueryStage +(58) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(57) InputAdapter +(59) InputAdapter Input [2]: [sum#X, count#X] -(58) InputIteratorTransformer +(60) InputIteratorTransformer Input [2]: [sum#X, count#X] -(59) RegularHashAggregateExecTransformer +(61) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(60) WholeStageCodegenTransformer (X) +(62) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(61) VeloxColumnarToRowExec +(63) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(62) Scan parquet +(64) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(63) Filter +(65) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(64) Project +(66) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(65) HashAggregate +(67) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(66) Exchange +(68) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(67) HashAggregate +(69) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(68) AdaptiveSparkPlan +(70) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt index 08e94f8df743..3d145f0c3bb8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt @@ -1,50 +1,51 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32), Statistics(X) - +- ColumnarExchange (31) - +- VeloxAppendBatches (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7), Statistics(X) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- VeloxAppendBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (52) - +- Exchange (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- ShuffledHashJoin LeftSemi BuildRight (46) - :- Exchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (53) + +- Exchange (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftSemi BuildRight (47) + :- Exchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -146,134 +147,138 @@ Input [2]: [o_orderpriority#X, count#X] Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(24) ColumnarExchange +(24) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(28) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(29) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(30) VeloxAppendBatches +(31) VeloxAppendBatches Input [2]: [o_orderpriority#X, order_count#X] Arguments: X -(31) ColumnarExchange +(32) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(35) SortExecTransformer +(36) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(37) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(41) Exchange +(42) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Scan parquet +(43) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(43) Filter +(44) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(44) Project +(45) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(45) Exchange +(46) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(47) Project +(48) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(48) HashAggregate +(49) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(49) Exchange +(50) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(51) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(51) Exchange +(52) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(53) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt index 888a5f38c53d..08e655f5aa81 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt @@ -1,126 +1,127 @@ == Physical Plan == -AdaptiveSparkPlan (145) +AdaptiveSparkPlan (146) +- == Final Plan == - VeloxColumnarToRowExec (105) - +- ^ SortExecTransformer (103) - +- ^ InputIteratorTransformer (102) - +- ShuffleQueryStage (100), Statistics(X) - +- ColumnarExchange (99) - +- VeloxAppendBatches (98) - +- ^ RegularHashAggregateExecTransformer (96) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ColumnarExchange (92) - +- ^ ProjectExecTransformer (90) - +- ^ FlushableHashAggregateExecTransformer (89) - +- ^ ProjectExecTransformer (88) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - :- ^ InputIteratorTransformer (77) - : +- ShuffleQueryStage (75), Statistics(X) - : +- ColumnarExchange (74) - : +- VeloxAppendBatches (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - : :- ^ InputIteratorTransformer (60) - : : +- ShuffleQueryStage (58), Statistics(X) - : : +- ColumnarExchange (57) - : : +- VeloxAppendBatches (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - : : :- ^ InputIteratorTransformer (43) - : : : +- ShuffleQueryStage (41), Statistics(X) - : : : +- ColumnarExchange (40) - : : : +- VeloxAppendBatches (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - : : : :- ^ InputIteratorTransformer (26) - : : : : +- ShuffleQueryStage (24), Statistics(X) - : : : : +- ColumnarExchange (23) - : : : : +- VeloxAppendBatches (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ InputIteratorTransformer (9) - : : : : : +- ShuffleQueryStage (7), Statistics(X) - : : : : : +- ColumnarExchange (6) - : : : : : +- VeloxAppendBatches (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (35) - : : : +- ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- VeloxAppendBatches (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ NoopFilter (28) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50), Statistics(X) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ NoopFilter (45) - : : +- ^ Scan parquet (44) - : +- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67), Statistics(X) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ NoopFilter (62) - : +- ^ Scan parquet (61) - +- ^ InputIteratorTransformer (86) - +- ShuffleQueryStage (84), Statistics(X) - +- ColumnarExchange (83) - +- VeloxAppendBatches (82) - +- ^ ProjectExecTransformer (80) - +- ^ NoopFilter (79) - +- ^ Scan parquet (78) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94), Statistics(X) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (144) - +- Exchange (143) - +- HashAggregate (142) - +- Exchange (141) - +- HashAggregate (140) - +- Project (139) - +- ShuffledHashJoin Inner BuildRight (138) - :- Exchange (133) - : +- Project (132) - : +- ShuffledHashJoin Inner BuildRight (131) - : :- Exchange (127) - : : +- Project (126) - : : +- ShuffledHashJoin Inner BuildRight (125) - : : :- Exchange (121) - : : : +- Project (120) - : : : +- ShuffledHashJoin Inner BuildRight (119) - : : : :- Exchange (115) - : : : : +- Project (114) - : : : : +- ShuffledHashJoin Inner BuildLeft (113) - : : : : :- Exchange (108) - : : : : : +- Filter (107) - : : : : : +- Scan parquet (106) - : : : : +- Exchange (112) - : : : : +- Project (111) - : : : : +- Filter (110) - : : : : +- Scan parquet (109) - : : : +- Exchange (118) - : : : +- Filter (117) - : : : +- Scan parquet (116) - : : +- Exchange (124) - : : +- Filter (123) - : : +- Scan parquet (122) - : +- Exchange (130) - : +- Filter (129) - : +- Scan parquet (128) - +- Exchange (137) - +- Project (136) - +- Filter (135) - +- Scan parquet (134) + Sort (145) + +- Exchange (144) + +- HashAggregate (143) + +- Exchange (142) + +- HashAggregate (141) + +- Project (140) + +- ShuffledHashJoin Inner BuildRight (139) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Project (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (138) + +- Project (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -498,250 +499,254 @@ Input [3]: [n_name#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(92) ColumnarExchange +(92) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(93) ShuffleQueryStage +(94) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(94) InputAdapter +(95) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(95) InputIteratorTransformer +(96) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(96) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(97) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(98) VeloxAppendBatches +(99) VeloxAppendBatches Input [2]: [n_name#X, revenue#X] Arguments: X -(99) ColumnarExchange +(100) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(100) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(101) InputAdapter +(102) InputAdapter Input [2]: [n_name#X, revenue#X] -(102) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(103) SortExecTransformer +(104) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(104) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(105) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(106) Scan parquet +(107) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(108) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(109) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Scan parquet +(110) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(110) Filter +(111) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(111) Project +(112) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(112) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(114) Project +(115) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(115) Exchange +(116) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) Scan parquet +(117) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(117) Filter +(118) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(118) Exchange +(119) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(120) Project +(121) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(121) Exchange +(122) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(123) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(123) Filter +(124) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(124) Exchange +(125) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(126) Project +(127) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(127) Exchange +(128) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(129) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(129) Filter +(130) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(130) Exchange +(131) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(132) Project +(133) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(133) Exchange +(134) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(134) Scan parquet +(135) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(135) Filter +(136) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(136) Project +(137) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(137) Exchange +(138) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(138) ShuffledHashJoin +(139) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(139) Project +(140) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(140) HashAggregate +(141) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(141) Exchange +(142) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) HashAggregate +(143) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(143) Exchange +(144) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(144) Sort +(145) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(145) AdaptiveSparkPlan +(146) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt index fbcc75a247d5..64624c791f72 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt @@ -7,7 +7,7 @@ AdaptiveSparkPlan (20) +- ShuffleQueryStage (8), Statistics(X) +- ColumnarExchange (7) +- VeloxAppendBatches (6) - +- ^ RegularHashAggregateExecTransformer (4) + +- ^ FlushableHashAggregateExecTransformer (4) +- ^ ProjectExecTransformer (3) +- ^ NoopFilter (2) +- ^ Scan parquet (1) @@ -35,7 +35,7 @@ Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * l_discount#X) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) RegularHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt index a7597d3dbe87..71ade94be21d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt @@ -1,120 +1,121 @@ == Physical Plan == -AdaptiveSparkPlan (138) +AdaptiveSparkPlan (139) +- == Final Plan == - VeloxColumnarToRowExec (100) - +- ^ SortExecTransformer (98) - +- ^ InputIteratorTransformer (97) - +- ShuffleQueryStage (95), Statistics(X) - +- ColumnarExchange (94) - +- VeloxAppendBatches (93) - +- ^ RegularHashAggregateExecTransformer (91) - +- ^ InputIteratorTransformer (90) - +- ShuffleQueryStage (88), Statistics(X) - +- ColumnarExchange (87) - +- ^ ProjectExecTransformer (85) - +- ^ FlushableHashAggregateExecTransformer (84) - +- ^ ProjectExecTransformer (83) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) - :- ^ InputIteratorTransformer (77) - : +- ShuffleQueryStage (75), Statistics(X) - : +- ColumnarExchange (74) - : +- VeloxAppendBatches (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - : :- ^ InputIteratorTransformer (60) - : : +- ShuffleQueryStage (58), Statistics(X) - : : +- ColumnarExchange (57) - : : +- VeloxAppendBatches (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - : : :- ^ InputIteratorTransformer (43) - : : : +- ShuffleQueryStage (41), Statistics(X) - : : : +- ColumnarExchange (40) - : : : +- VeloxAppendBatches (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : :- ^ InputIteratorTransformer (26) - : : : : +- ShuffleQueryStage (24), Statistics(X) - : : : : +- ColumnarExchange (23) - : : : : +- VeloxAppendBatches (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ InputIteratorTransformer (9) - : : : : : +- ShuffleQueryStage (7), Statistics(X) - : : : : : +- ColumnarExchange (6) - : : : : : +- VeloxAppendBatches (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (35) - : : : +- ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- VeloxAppendBatches (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ NoopFilter (28) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50), Statistics(X) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ NoopFilter (45) - : : +- ^ Scan parquet (44) - : +- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67), Statistics(X) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ NoopFilter (62) - : +- ^ Scan parquet (61) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ReusedExchange (78) + VeloxColumnarToRowExec (101) + +- ^ SortExecTransformer (99) + +- ^ InputIteratorTransformer (98) + +- ShuffleQueryStage (96), Statistics(X) + +- ColumnarExchange (95) + +- VeloxAppendBatches (94) + +- ^ RegularHashAggregateExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- VeloxAppendBatches (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ReusedExchange (78) +- == Initial Plan == - Sort (137) - +- Exchange (136) - +- HashAggregate (135) - +- Exchange (134) - +- HashAggregate (133) - +- Project (132) - +- ShuffledHashJoin Inner BuildRight (131) - :- Exchange (127) - : +- Project (126) - : +- ShuffledHashJoin Inner BuildRight (125) - : :- Exchange (121) - : : +- Project (120) - : : +- ShuffledHashJoin Inner BuildRight (119) - : : :- Exchange (115) - : : : +- Project (114) - : : : +- ShuffledHashJoin Inner BuildRight (113) - : : : :- Exchange (109) - : : : : +- Project (108) - : : : : +- ShuffledHashJoin Inner BuildLeft (107) - : : : : :- Exchange (103) - : : : : : +- Filter (102) - : : : : : +- Scan parquet (101) - : : : : +- Exchange (106) - : : : : +- Filter (105) - : : : : +- Scan parquet (104) - : : : +- Exchange (112) - : : : +- Filter (111) - : : : +- Scan parquet (110) - : : +- Exchange (118) - : : +- Filter (117) - : : +- Scan parquet (116) - : +- Exchange (124) - : +- Filter (123) - : +- Scan parquet (122) - +- Exchange (130) - +- Filter (129) - +- Scan parquet (128) + Sort (138) + +- Exchange (137) + +- HashAggregate (136) + +- Exchange (135) + +- HashAggregate (134) + +- Project (133) + +- ShuffledHashJoin Inner BuildRight (132) + :- Exchange (128) + : +- Project (127) + : +- ShuffledHashJoin Inner BuildRight (126) + : :- Exchange (122) + : : +- Project (121) + : : +- ShuffledHashJoin Inner BuildRight (120) + : : :- Exchange (116) + : : : +- Project (115) + : : : +- ShuffledHashJoin Inner BuildRight (114) + : : : :- Exchange (110) + : : : : +- Project (109) + : : : : +- ShuffledHashJoin Inner BuildLeft (108) + : : : : :- Exchange (104) + : : : : : +- Filter (103) + : : : : : +- Scan parquet (102) + : : : : +- Exchange (107) + : : : : +- Filter (106) + : : : : +- Scan parquet (105) + : : : +- Exchange (113) + : : : +- Filter (112) + : : : +- Scan parquet (111) + : : +- Exchange (119) + : : +- Filter (118) + : : +- Scan parquet (117) + : +- Exchange (125) + : +- Filter (124) + : +- Scan parquet (123) + +- Exchange (131) + +- Filter (130) + +- Scan parquet (129) (1) Scan parquet @@ -468,242 +469,246 @@ Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(87) ColumnarExchange +(87) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(88) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(88) ShuffleQueryStage +(89) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(89) InputAdapter +(90) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(90) InputIteratorTransformer +(91) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(91) RegularHashAggregateExecTransformer +(92) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(92) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(93) VeloxAppendBatches +(94) VeloxAppendBatches Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(94) ColumnarExchange +(95) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(95) ShuffleQueryStage +(96) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(96) InputAdapter +(97) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(97) InputIteratorTransformer +(98) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(98) SortExecTransformer +(99) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(99) WholeStageCodegenTransformer (X) +(100) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(100) VeloxColumnarToRowExec +(101) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(101) Scan parquet +(102) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(102) Filter +(103) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(103) Exchange +(104) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) Scan parquet +(105) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(105) Filter +(106) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(106) Exchange +(107) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(108) Project +(109) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(109) Exchange +(110) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(112) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(114) Project +(115) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(115) Exchange +(116) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) Scan parquet +(117) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(117) Filter +(118) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(118) Exchange +(119) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(120) Project +(121) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(121) Exchange +(122) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(123) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(123) Filter +(124) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(124) Exchange +(125) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(126) Project +(127) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(127) Exchange +(128) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(129) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(129) Filter +(130) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(130) Exchange +(131) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(132) Project +(133) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(133) HashAggregate +(134) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(134) Exchange +(135) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(135) HashAggregate +(136) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(136) Exchange +(137) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) Sort +(138) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(138) AdaptiveSparkPlan +(139) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt index 2e5a862f832f..ddeab25c4569 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt @@ -1,165 +1,166 @@ == Physical Plan == -AdaptiveSparkPlan (192) +AdaptiveSparkPlan (193) +- == Final Plan == - VeloxColumnarToRowExec (140) - +- ^ SortExecTransformer (138) - +- ^ InputIteratorTransformer (137) - +- ShuffleQueryStage (135), Statistics(X) - +- ColumnarExchange (134) - +- VeloxAppendBatches (133) - +- ^ ProjectExecTransformer (131) - +- ^ RegularHashAggregateExecTransformer (130) - +- ^ InputIteratorTransformer (129) - +- ShuffleQueryStage (127), Statistics(X) - +- ColumnarExchange (126) - +- ^ ProjectExecTransformer (124) - +- ^ FlushableHashAggregateExecTransformer (123) - +- ^ ProjectExecTransformer (122) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) - :- ^ InputIteratorTransformer (111) - : +- ShuffleQueryStage (109), Statistics(X) - : +- ColumnarExchange (108) - : +- VeloxAppendBatches (107) - : +- ^ ProjectExecTransformer (105) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) - : :- ^ InputIteratorTransformer (94) - : : +- ShuffleQueryStage (92), Statistics(X) - : : +- ColumnarExchange (91) - : : +- VeloxAppendBatches (90) - : : +- ^ ProjectExecTransformer (88) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - : : :- ^ InputIteratorTransformer (77) - : : : +- ShuffleQueryStage (75), Statistics(X) - : : : +- ColumnarExchange (74) - : : : +- VeloxAppendBatches (73) - : : : +- ^ ProjectExecTransformer (71) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) - : : : :- ^ InputIteratorTransformer (60) - : : : : +- ShuffleQueryStage (58), Statistics(X) - : : : : +- ColumnarExchange (57) - : : : : +- VeloxAppendBatches (56) - : : : : +- ^ ProjectExecTransformer (54) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) - : : : : :- ^ InputIteratorTransformer (43) - : : : : : +- ShuffleQueryStage (41), Statistics(X) - : : : : : +- ColumnarExchange (40) - : : : : : +- VeloxAppendBatches (39) - : : : : : +- ^ ProjectExecTransformer (37) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : : : :- ^ InputIteratorTransformer (26) - : : : : : : +- ShuffleQueryStage (24), Statistics(X) - : : : : : : +- ColumnarExchange (23) - : : : : : : +- VeloxAppendBatches (22) - : : : : : : +- ^ ProjectExecTransformer (20) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : : : :- ^ InputIteratorTransformer (9) - : : : : : : : +- ShuffleQueryStage (7), Statistics(X) - : : : : : : : +- ColumnarExchange (6) - : : : : : : : +- VeloxAppendBatches (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (18) - : : : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : : : +- ColumnarExchange (15) - : : : : : : +- VeloxAppendBatches (14) - : : : : : : +- ^ ProjectExecTransformer (12) - : : : : : : +- ^ NoopFilter (11) - : : : : : : +- ^ Scan parquet (10) - : : : : : +- ^ InputIteratorTransformer (35) - : : : : : +- ShuffleQueryStage (33), Statistics(X) - : : : : : +- ColumnarExchange (32) - : : : : : +- VeloxAppendBatches (31) - : : : : : +- ^ ProjectExecTransformer (29) - : : : : : +- ^ NoopFilter (28) - : : : : : +- ^ Scan parquet (27) - : : : : +- ^ InputIteratorTransformer (52) - : : : : +- ShuffleQueryStage (50), Statistics(X) - : : : : +- ColumnarExchange (49) - : : : : +- VeloxAppendBatches (48) - : : : : +- ^ ProjectExecTransformer (46) - : : : : +- ^ NoopFilter (45) - : : : : +- ^ Scan parquet (44) - : : : +- ^ InputIteratorTransformer (69) - : : : +- ShuffleQueryStage (67), Statistics(X) - : : : +- ColumnarExchange (66) - : : : +- VeloxAppendBatches (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ NoopFilter (62) - : : : +- ^ Scan parquet (61) - : : +- ^ InputIteratorTransformer (86) - : : +- ShuffleQueryStage (84), Statistics(X) - : : +- ColumnarExchange (83) - : : +- VeloxAppendBatches (82) - : : +- ^ ProjectExecTransformer (80) - : : +- ^ NoopFilter (79) - : : +- ^ Scan parquet (78) - : +- ^ InputIteratorTransformer (103) - : +- ShuffleQueryStage (101), Statistics(X) - : +- ColumnarExchange (100) - : +- VeloxAppendBatches (99) - : +- ^ ProjectExecTransformer (97) - : +- ^ NoopFilter (96) - : +- ^ Scan parquet (95) - +- ^ InputIteratorTransformer (120) - +- ShuffleQueryStage (118), Statistics(X) - +- ColumnarExchange (117) - +- VeloxAppendBatches (116) - +- ^ ProjectExecTransformer (114) - +- ^ NoopFilter (113) - +- ^ Scan parquet (112) + VeloxColumnarToRowExec (141) + +- ^ SortExecTransformer (139) + +- ^ InputIteratorTransformer (138) + +- ShuffleQueryStage (136), Statistics(X) + +- ColumnarExchange (135) + +- VeloxAppendBatches (134) + +- ^ ProjectExecTransformer (132) + +- ^ RegularHashAggregateExecTransformer (131) + +- ^ InputIteratorTransformer (130) + +- ShuffleQueryStage (128), Statistics(X) + +- ColumnarExchange (127) + +- VeloxAppendBatches (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109), Statistics(X) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92), Statistics(X) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75), Statistics(X) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58), Statistics(X) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41), Statistics(X) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ NoopFilter (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33), Statistics(X) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ NoopFilter (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50), Statistics(X) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ NoopFilter (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67), Statistics(X) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ NoopFilter (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ NoopFilter (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ NoopFilter (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118), Statistics(X) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ NoopFilter (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (191) - +- Exchange (190) - +- HashAggregate (189) - +- Exchange (188) - +- HashAggregate (187) - +- Project (186) - +- ShuffledHashJoin Inner BuildRight (185) - :- Exchange (180) - : +- Project (179) - : +- ShuffledHashJoin Inner BuildRight (178) - : :- Exchange (174) - : : +- Project (173) - : : +- ShuffledHashJoin Inner BuildRight (172) - : : :- Exchange (168) - : : : +- Project (167) - : : : +- ShuffledHashJoin Inner BuildRight (166) - : : : :- Exchange (162) - : : : : +- Project (161) - : : : : +- ShuffledHashJoin Inner BuildRight (160) - : : : : :- Exchange (156) - : : : : : +- Project (155) - : : : : : +- ShuffledHashJoin Inner BuildRight (154) - : : : : : :- Exchange (150) - : : : : : : +- Project (149) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (148) - : : : : : : :- Exchange (144) - : : : : : : : +- Project (143) - : : : : : : : +- Filter (142) - : : : : : : : +- Scan parquet (141) - : : : : : : +- Exchange (147) - : : : : : : +- Filter (146) - : : : : : : +- Scan parquet (145) - : : : : : +- Exchange (153) - : : : : : +- Filter (152) - : : : : : +- Scan parquet (151) - : : : : +- Exchange (159) - : : : : +- Filter (158) - : : : : +- Scan parquet (157) - : : : +- Exchange (165) - : : : +- Filter (164) - : : : +- Scan parquet (163) - : : +- Exchange (171) - : : +- Filter (170) - : : +- Scan parquet (169) - : +- Exchange (177) - : +- Filter (176) - : +- Scan parquet (175) - +- Exchange (184) - +- Project (183) - +- Filter (182) - +- Scan parquet (181) + Sort (192) + +- Exchange (191) + +- HashAggregate (190) + +- Exchange (189) + +- HashAggregate (188) + +- Project (187) + +- ShuffledHashJoin Inner BuildRight (186) + :- Exchange (181) + : +- Project (180) + : +- ShuffledHashJoin Inner BuildRight (179) + : :- Exchange (175) + : : +- Project (174) + : : +- ShuffledHashJoin Inner BuildRight (173) + : : :- Exchange (169) + : : : +- Project (168) + : : : +- ShuffledHashJoin Inner BuildRight (167) + : : : :- Exchange (163) + : : : : +- Project (162) + : : : : +- ShuffledHashJoin Inner BuildRight (161) + : : : : :- Exchange (157) + : : : : : +- Project (156) + : : : : : +- ShuffledHashJoin Inner BuildRight (155) + : : : : : :- Exchange (151) + : : : : : : +- Project (150) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (149) + : : : : : : :- Exchange (145) + : : : : : : : +- Project (144) + : : : : : : : +- Filter (143) + : : : : : : : +- Scan parquet (142) + : : : : : : +- Exchange (148) + : : : : : : +- Filter (147) + : : : : : : +- Scan parquet (146) + : : : : : +- Exchange (154) + : : : : : +- Filter (153) + : : : : : +- Scan parquet (152) + : : : : +- Exchange (160) + : : : : +- Filter (159) + : : : : +- Scan parquet (158) + : : : +- Exchange (166) + : : : +- Filter (165) + : : : +- Scan parquet (164) + : : +- Exchange (172) + : : +- Filter (171) + : : +- Scan parquet (170) + : +- Exchange (178) + : +- Filter (177) + : +- Scan parquet (176) + +- Exchange (185) + +- Project (184) + +- Filter (183) + +- Scan parquet (182) (1) Scan parquet @@ -675,312 +676,316 @@ Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(126) ColumnarExchange +(126) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(127) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(127) ShuffleQueryStage +(128) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(128) InputAdapter +(129) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(129) InputIteratorTransformer +(130) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(130) RegularHashAggregateExecTransformer +(131) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(131) ProjectExecTransformer +(132) ProjectExecTransformer Output [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(132) WholeStageCodegenTransformer (X) +(133) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(133) VeloxAppendBatches +(134) VeloxAppendBatches Input [2]: [o_year#X, mkt_share#X] Arguments: X -(134) ColumnarExchange +(135) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(135) ShuffleQueryStage +(136) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(136) InputAdapter +(137) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(137) InputIteratorTransformer +(138) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(138) SortExecTransformer +(139) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(139) WholeStageCodegenTransformer (X) +(140) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(140) VeloxColumnarToRowExec +(141) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(141) Scan parquet +(142) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(142) Filter +(143) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(143) Project +(144) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(144) Exchange +(145) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) Scan parquet +(146) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(146) Filter +(147) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(147) Exchange +(148) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) ShuffledHashJoin +(149) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(149) Project +(150) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(150) Exchange +(151) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) Scan parquet +(152) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(152) Filter +(153) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(153) Exchange +(154) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) ShuffledHashJoin +(155) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(155) Project +(156) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(156) Exchange +(157) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) Scan parquet +(158) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(158) Filter +(159) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(159) Exchange +(160) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) ShuffledHashJoin +(161) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(161) Project +(162) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(162) Exchange +(163) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) Scan parquet +(164) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(164) Filter +(165) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(165) Exchange +(166) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) ShuffledHashJoin +(167) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(167) Project +(168) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(168) Exchange +(169) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(169) Scan parquet +(170) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(170) Filter +(171) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(171) Exchange +(172) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(172) ShuffledHashJoin +(173) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(173) Project +(174) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(174) Exchange +(175) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(175) Scan parquet +(176) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(176) Filter +(177) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(177) Exchange +(178) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(178) ShuffledHashJoin +(179) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(179) Project +(180) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(180) Exchange +(181) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(181) Scan parquet +(182) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(182) Filter +(183) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(183) Project +(184) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(184) Exchange +(185) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(185) ShuffledHashJoin +(186) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(186) Project +(187) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(187) HashAggregate +(188) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(188) Exchange +(189) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(189) HashAggregate +(190) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(190) Exchange +(191) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(191) Sort +(192) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(192) AdaptiveSparkPlan +(193) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt index bbc0a7042023..634e3516a710 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt @@ -1,125 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (144) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (105) - +- ^ SortExecTransformer (103) - +- ^ InputIteratorTransformer (102) - +- ShuffleQueryStage (100), Statistics(X) - +- ColumnarExchange (99) - +- VeloxAppendBatches (98) - +- ^ RegularHashAggregateExecTransformer (96) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ColumnarExchange (92) - +- ^ ProjectExecTransformer (90) - +- ^ FlushableHashAggregateExecTransformer (89) - +- ^ ProjectExecTransformer (88) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - :- ^ InputIteratorTransformer (77) - : +- ShuffleQueryStage (75), Statistics(X) - : +- ColumnarExchange (74) - : +- VeloxAppendBatches (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) - : :- ^ InputIteratorTransformer (60) - : : +- ShuffleQueryStage (58), Statistics(X) - : : +- ColumnarExchange (57) - : : +- VeloxAppendBatches (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) - : : :- ^ InputIteratorTransformer (43) - : : : +- ShuffleQueryStage (41), Statistics(X) - : : : +- ColumnarExchange (40) - : : : +- VeloxAppendBatches (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : :- ^ InputIteratorTransformer (26) - : : : : +- ShuffleQueryStage (24), Statistics(X) - : : : : +- ColumnarExchange (23) - : : : : +- VeloxAppendBatches (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ InputIteratorTransformer (9) - : : : : : +- ShuffleQueryStage (7), Statistics(X) - : : : : : +- ColumnarExchange (6) - : : : : : +- VeloxAppendBatches (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (35) - : : : +- ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- VeloxAppendBatches (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ NoopFilter (28) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50), Statistics(X) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ NoopFilter (45) - : : +- ^ Scan parquet (44) - : +- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67), Statistics(X) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ NoopFilter (62) - : +- ^ Scan parquet (61) - +- ^ InputIteratorTransformer (86) - +- ShuffleQueryStage (84), Statistics(X) - +- ColumnarExchange (83) - +- VeloxAppendBatches (82) - +- ^ ProjectExecTransformer (80) - +- ^ NoopFilter (79) - +- ^ Scan parquet (78) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94), Statistics(X) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (143) - +- Exchange (142) - +- HashAggregate (141) - +- Exchange (140) - +- HashAggregate (139) - +- Project (138) - +- ShuffledHashJoin Inner BuildRight (137) - :- Exchange (133) - : +- Project (132) - : +- ShuffledHashJoin Inner BuildRight (131) - : :- Exchange (127) - : : +- Project (126) - : : +- ShuffledHashJoin Inner BuildRight (125) - : : :- Exchange (121) - : : : +- Project (120) - : : : +- ShuffledHashJoin Inner BuildRight (119) - : : : :- Exchange (115) - : : : : +- Project (114) - : : : : +- ShuffledHashJoin Inner BuildLeft (113) - : : : : :- Exchange (109) - : : : : : +- Project (108) - : : : : : +- Filter (107) - : : : : : +- Scan parquet (106) - : : : : +- Exchange (112) - : : : : +- Filter (111) - : : : : +- Scan parquet (110) - : : : +- Exchange (118) - : : : +- Filter (117) - : : : +- Scan parquet (116) - : : +- Exchange (124) - : : +- Filter (123) - : : +- Scan parquet (122) - : +- Exchange (130) - : +- Filter (129) - : +- Scan parquet (128) - +- Exchange (136) - +- Filter (135) - +- Scan parquet (134) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (110) + : : : : : +- Project (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Filter (112) + : : : : +- Scan parquet (111) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -497,246 +498,250 @@ Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(92) ColumnarExchange +(92) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(93) ShuffleQueryStage +(94) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(94) InputAdapter +(95) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(95) InputIteratorTransformer +(96) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(96) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(97) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(98) VeloxAppendBatches +(99) VeloxAppendBatches Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(99) ColumnarExchange +(100) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(100) ShuffleQueryStage +(101) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(101) InputAdapter +(102) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(102) InputIteratorTransformer +(103) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(103) SortExecTransformer +(104) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(104) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(105) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(106) Scan parquet +(107) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(107) Filter +(108) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(108) Project +(109) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(109) Exchange +(110) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(111) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(112) Exchange +(113) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(114) Project +(115) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(115) Exchange +(116) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) Scan parquet +(117) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(117) Filter +(118) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(118) Exchange +(119) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(120) Project +(121) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(121) Exchange +(122) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(123) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(123) Filter +(124) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(124) Exchange +(125) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(126) Project +(127) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(127) Exchange +(128) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(129) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(129) Filter +(130) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(130) Exchange +(131) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(132) Project +(133) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(133) Exchange +(134) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(134) Scan parquet +(135) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(135) Filter +(136) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(136) Exchange +(137) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(138) Project +(139) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(139) HashAggregate +(140) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(140) Exchange +(141) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(141) HashAggregate +(142) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(142) Exchange +(143) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(143) Sort +(144) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(144) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/1.txt index 1c8670c0fea4..63b7d317f3cf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (29) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (28) - +- Exchange (27) - +- HashAggregate (26) - +- Exchange (25) - +- HashAggregate (24) - +- Project (23) - +- Filter (22) - +- Scan parquet (21) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -57,101 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) VeloxAppendBatches +(14) VeloxAppendBatches Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) ColumnarExchange +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(18) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(20) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(21) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(22) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(23) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(24) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(25) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(27) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(29) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt index 8233c7669206..c5fcd91867cb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt @@ -1,84 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (66) - +- TakeOrderedAndProjectExecTransformer (65) - +- ^ ProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59) - +- ColumnarExchange (58) - +- ^ ProjectExecTransformer (56) - +- ^ FlushableHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - :- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41) - : +- ColumnarExchange (40) - : +- VeloxAppendBatches (39) - : +- ^ ProjectExecTransformer (37) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - : :- ^ InputIteratorTransformer (26) - : : +- ShuffleQueryStage (24) - : : +- ColumnarExchange (23) - : : +- VeloxAppendBatches (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : : :- ^ InputIteratorTransformer (9) - : : : +- ShuffleQueryStage (7) - : : : +- ColumnarExchange (6) - : : : +- VeloxAppendBatches (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (18) - : : +- ShuffleQueryStage (16) - : : +- ColumnarExchange (15) - : : +- VeloxAppendBatches (14) - : : +- ^ ProjectExecTransformer (12) - : : +- ^ FilterExecTransformer (11) - : : +- ^ Scan parquet (10) - : +- ^ InputIteratorTransformer (35) - : +- ShuffleQueryStage (33) - : +- ColumnarExchange (32) - : +- VeloxAppendBatches (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ FilterExecTransformer (28) - : +- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50) - +- ColumnarExchange (49) - +- VeloxAppendBatches (48) - +- ^ ProjectExecTransformer (46) - +- ^ FilterExecTransformer (45) - +- ^ Scan parquet (44) + VeloxColumnarToRowExec (67) + +- TakeOrderedAndProjectExecTransformer (66) + +- ^ ProjectExecTransformer (64) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FilterExecTransformer (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (92) - +- HashAggregate (91) - +- Exchange (90) - +- HashAggregate (89) - +- Project (88) - +- ShuffledHashJoin Inner BuildRight (87) - :- Exchange (83) - : +- Project (82) - : +- ShuffledHashJoin Inner BuildRight (81) - : :- Exchange (76) - : : +- Project (75) - : : +- ShuffledHashJoin Inner BuildRight (74) - : : :- Exchange (69) - : : : +- Filter (68) - : : : +- Scan parquet (67) - : : +- Exchange (73) - : : +- Project (72) - : : +- Filter (71) - : : +- Scan parquet (70) - : +- Exchange (80) - : +- Project (79) - : +- Filter (78) - : +- Scan parquet (77) - +- Exchange (86) - +- Filter (85) - +- Scan parquet (84) + TakeOrderedAndProject (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- ShuffledHashJoin Inner BuildRight (88) + :- Exchange (84) + : +- Project (83) + : +- ShuffledHashJoin Inner BuildRight (82) + : :- Exchange (77) + : : +- Project (76) + : : +- ShuffledHashJoin Inner BuildRight (75) + : : :- Exchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- Exchange (74) + : : +- Project (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (81) + : +- Project (80) + : +- Filter (79) + : +- Scan parquet (78) + +- Exchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -315,167 +316,171 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(58) ColumnarExchange +(58) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(59) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(60) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(60) InputAdapter +(61) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(61) InputIteratorTransformer +(62) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(62) RegularHashAggregateExecTransformer +(63) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(63) ProjectExecTransformer +(64) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(65) TakeOrderedAndProjectExecTransformer +(66) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(66) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) Scan parquet +(68) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(68) Filter +(69) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(69) Exchange +(70) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(70) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(72) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(73) Exchange +(74) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) ShuffledHashJoin +(75) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(75) Project +(76) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(76) Exchange +(77) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Scan parquet +(78) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(78) Filter +(79) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(79) Project +(80) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(80) Exchange +(81) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(82) Project +(83) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(83) Exchange +(84) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(86) Exchange +(87) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) ShuffledHashJoin +(88) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) Project +(89) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(89) HashAggregate +(90) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(90) Exchange +(91) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(92) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(92) TakeOrderedAndProject +(93) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(93) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt index bc664f216eb3..59de06707aad 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt @@ -1,70 +1,71 @@ == Physical Plan == -AdaptiveSparkPlan (77) +AdaptiveSparkPlan (78) +- == Final Plan == - VeloxColumnarToRowExec (55) - +- ^ SortExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50) - +- ColumnarExchange (49) - +- VeloxAppendBatches (48) - +- ^ FilterExecTransformer (46) - +- ^ RegularHashAggregateExecTransformer (45) - +- ^ InputIteratorTransformer (44) - +- ShuffleQueryStage (42) - +- ColumnarExchange (41) - +- ^ ProjectExecTransformer (39) - +- ^ FlushableHashAggregateExecTransformer (38) - +- ^ ProjectExecTransformer (37) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - :- ^ InputIteratorTransformer (26) - : +- ShuffleQueryStage (24) - : +- ColumnarExchange (23) - : +- VeloxAppendBatches (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : :- ^ InputIteratorTransformer (9) - : : +- ShuffleQueryStage (7) - : : +- ColumnarExchange (6) - : : +- VeloxAppendBatches (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (18) - : +- ShuffleQueryStage (16) - : +- ColumnarExchange (15) - : +- VeloxAppendBatches (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FilterExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33) - +- ColumnarExchange (32) - +- VeloxAppendBatches (31) - +- ^ ProjectExecTransformer (29) - +- ^ FilterExecTransformer (28) - +- ^ Scan parquet (27) + VeloxColumnarToRowExec (56) + +- ^ SortExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ShuffleQueryStage (51) + +- ColumnarExchange (50) + +- VeloxAppendBatches (49) + +- ^ FilterExecTransformer (47) + +- ^ RegularHashAggregateExecTransformer (46) + +- ^ InputIteratorTransformer (45) + +- ShuffleQueryStage (43) + +- ColumnarExchange (42) + +- VeloxAppendBatches (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (76) - +- Exchange (75) - +- Filter (74) - +- HashAggregate (73) - +- Exchange (72) - +- HashAggregate (71) - +- Project (70) - +- ShuffledHashJoin Inner BuildRight (69) - :- Exchange (64) - : +- Project (63) - : +- ShuffledHashJoin Inner BuildRight (62) - : :- Exchange (58) - : : +- Filter (57) - : : +- Scan parquet (56) - : +- Exchange (61) - : +- Filter (60) - : +- Scan parquet (59) - +- Exchange (68) - +- Project (67) - +- Filter (66) - +- Scan parquet (65) + Sort (77) + +- Exchange (76) + +- Filter (75) + +- HashAggregate (74) + +- Exchange (73) + +- HashAggregate (72) + +- Project (71) + +- ShuffledHashJoin Inner BuildRight (70) + :- Exchange (65) + : +- Project (64) + : +- ShuffledHashJoin Inner BuildRight (63) + : :- Exchange (59) + : : +- Filter (58) + : : +- Scan parquet (57) + : +- Exchange (62) + : +- Filter (61) + : +- Scan parquet (60) + +- Exchange (69) + +- Project (68) + +- Filter (67) + +- Scan parquet (66) (1) Scan parquet @@ -233,165 +234,169 @@ Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(41) ColumnarExchange +(41) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(42) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(42) ShuffleQueryStage +(43) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(43) InputAdapter +(44) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(44) InputIteratorTransformer +(45) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(45) RegularHashAggregateExecTransformer +(46) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(46) FilterExecTransformer +(47) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(47) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(48) VeloxAppendBatches +(49) VeloxAppendBatches Input [2]: [ps_partkey#X, value#X] Arguments: X -(49) ColumnarExchange +(50) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(51) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(51) InputAdapter +(52) InputAdapter Input [2]: [ps_partkey#X, value#X] -(52) InputIteratorTransformer +(53) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(53) SortExecTransformer +(54) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(54) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(55) VeloxColumnarToRowExec +(56) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(56) Scan parquet +(57) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(57) Filter +(58) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(58) Exchange +(59) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) Scan parquet +(60) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(60) Filter +(61) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(61) Exchange +(62) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) ShuffledHashJoin +(63) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(63) Project +(64) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(64) Exchange +(65) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) Scan parquet +(66) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(67) Project +(68) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(68) Exchange +(69) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(69) ShuffledHashJoin +(70) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(70) Project +(71) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(71) HashAggregate +(72) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(72) Exchange +(73) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(73) HashAggregate +(74) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(74) Filter +(75) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(75) Exchange +(76) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) Sort +(77) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(77) AdaptiveSparkPlan +(78) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt index 2004f617a1d8..a8ac5d0d2c1b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt @@ -1,49 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32) - +- ColumnarExchange (31) - +- VeloxAppendBatches (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- VeloxAppendBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ FilterExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- Project (46) - +- ShuffledHashJoin Inner BuildLeft (45) - :- Exchange (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Exchange (44) - +- Project (43) - +- Filter (42) - +- Scan parquet (41) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin Inner BuildLeft (46) + :- Exchange (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -144,129 +145,133 @@ Input [3]: [l_shipmode#X, sum#X, sum#X] Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(24) ColumnarExchange +(24) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(25) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(28) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(29) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(30) VeloxAppendBatches +(31) VeloxAppendBatches Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(31) ColumnarExchange +(32) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) SortExecTransformer +(36) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(37) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(38) Scan parquet +(39) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(40) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(42) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(42) Filter +(43) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(43) Project +(44) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(44) Exchange +(45) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(46) Project +(47) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(47) HashAggregate +(48) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(48) Exchange +(49) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(50) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(50) Exchange +(51) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(52) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(52) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt index 0e57911ad3ca..d65867ecf822 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (55) +AdaptiveSparkPlan (56) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- ^ SortExecTransformer (37) - +- ^ InputIteratorTransformer (36) - +- ShuffleQueryStage (34) - +- ColumnarExchange (33) - +- VeloxAppendBatches (32) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- VeloxAppendBatches (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (54) - +- Exchange (53) - +- HashAggregate (52) - +- Exchange (51) - +- HashAggregate (50) - +- HashAggregate (49) - +- HashAggregate (48) - +- Project (47) - +- ShuffledHashJoin LeftOuter BuildRight (46) - :- Exchange (41) - : +- Scan parquet (40) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftOuter BuildRight (47) + :- Exchange (42) + : +- Scan parquet (41) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -160,138 +161,142 @@ Input [2]: [c_count#X, count#X] Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [2]: [c_count#X, count#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(32) VeloxAppendBatches +(33) VeloxAppendBatches Input [2]: [c_count#X, custdist#X] Arguments: X -(33) ColumnarExchange +(34) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(34) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(35) InputAdapter +(36) InputAdapter Input [2]: [c_count#X, custdist#X] -(36) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(37) SortExecTransformer +(38) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(38) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(39) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(40) Scan parquet +(41) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(41) Exchange +(42) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Scan parquet +(43) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(43) Filter +(44) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(44) Project +(45) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(45) Exchange +(46) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(47) Project +(48) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(48) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(49) HashAggregate +(50) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(50) HashAggregate +(51) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(51) Exchange +(52) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) HashAggregate +(53) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(53) Exchange +(54) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Sort +(55) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(55) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt index 40bb7d7d7983..0d21930825c7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt @@ -1,14 +1,14 @@ == Physical Plan == -AdaptiveSparkPlan (47) +AdaptiveSparkPlan (48) +- == Final Plan == - VeloxColumnarToRowExec (32) - +- ^ SortExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- VeloxAppendBatches (25) - +- ^ ProjectExecTransformer (23) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (22) + VeloxColumnarToRowExec (33) + +- ^ SortExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (23) :- ^ InputIteratorTransformer (9) : +- ShuffleQueryStage (7) : +- ColumnarExchange (6) @@ -16,31 +16,32 @@ AdaptiveSparkPlan (47) : +- ^ ProjectExecTransformer (3) : +- ^ FilterExecTransformer (2) : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ FilterExecTransformer (11) - +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (46) - +- Exchange (45) - +- Project (44) - +- ShuffledHashJoin Inner BuildLeft (43) - :- Exchange (35) - : +- Filter (34) - : +- Scan parquet (33) - +- Filter (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- Filter (37) - +- Scan parquet (36) + Sort (47) + +- Exchange (46) + +- Project (45) + +- ShuffledHashJoin Inner BuildLeft (44) + :- Exchange (36) + : +- Filter (35) + : +- Scan parquet (34) + +- Filter (43) + +- HashAggregate (42) + +- Exchange (41) + +- HashAggregate (40) + +- Project (39) + +- Filter (38) + +- Scan parquet (37) (1) Scan parquet @@ -110,142 +111,146 @@ Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(16) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(21) FilterExecTransformer +(22) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(22) ShuffledHashJoinExecTransformer +(23) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(23) ProjectExecTransformer +(24) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(24) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(25) VeloxAppendBatches +(26) VeloxAppendBatches Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) ColumnarExchange +(27) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(30) SortExecTransformer +(31) SortExecTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(32) VeloxColumnarToRowExec +(33) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(33) Scan parquet +(34) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(34) Filter +(35) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(35) Exchange +(36) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) Scan parquet +(37) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(37) Filter +(38) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(38) Project +(39) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(39) HashAggregate +(40) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(40) Exchange +(41) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) HashAggregate +(42) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(42) Filter +(43) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(43) ShuffledHashJoin +(44) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(44) Project +(45) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(45) Exchange +(46) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) Sort +(47) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(47) AdaptiveSparkPlan +(48) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt index 90afc1a3db17..cd3e53ad7bd6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt @@ -1,62 +1,64 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (45) - +- ^ SortExecTransformer (43) - +- ^ InputIteratorTransformer (42) - +- ShuffleQueryStage (40) - +- ColumnarExchange (39) - +- VeloxAppendBatches (38) - +- ^ RegularHashAggregateExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33) - +- ColumnarExchange (32) - +- ^ ProjectExecTransformer (30) - +- ^ FlushableHashAggregateExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- VeloxAppendBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ FilterExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (47) + +- ^ SortExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42) + +- ColumnarExchange (41) + +- VeloxAppendBatches (40) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ ProjectExecTransformer (31) + +- ^ FlushableHashAggregateExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (66) - +- Exchange (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- Project (58) - +- ShuffledHashJoin Inner BuildRight (57) - :- Exchange (53) - : +- BroadcastHashJoin LeftAnti BuildRight (52) - : :- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (51) - : +- Project (50) - : +- Filter (49) - : +- Scan parquet (48) - +- Exchange (56) - +- Filter (55) - +- Scan parquet (54) + Sort (68) + +- Exchange (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- HashAggregate (63) + +- Exchange (62) + +- HashAggregate (61) + +- Project (60) + +- ShuffledHashJoin Inner BuildRight (59) + :- Exchange (55) + : +- BroadcastHashJoin LeftAnti BuildRight (54) + : :- Filter (49) + : : +- Scan parquet (48) + : +- BroadcastExchange (53) + : +- Project (52) + : +- Filter (51) + : +- Scan parquet (50) + +- Exchange (58) + +- Filter (57) + +- Scan parquet (56) (1) Scan parquet @@ -157,203 +159,211 @@ Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(24) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(28) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(29) FlushableHashAggregateExecTransformer +(30) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(30) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(32) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(34) InputAdapter +(36) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(35) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(36) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(37) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(38) VeloxAppendBatches +(40) VeloxAppendBatches Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(39) ColumnarExchange +(41) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(42) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(41) InputAdapter +(43) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(42) InputIteratorTransformer +(44) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) SortExecTransformer +(45) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(44) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(45) VeloxColumnarToRowExec +(47) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(46) Scan parquet +(48) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(47) Filter +(49) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(48) Scan parquet +(50) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(49) Filter +(51) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(50) Project +(52) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(51) BroadcastExchange +(53) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(52) BroadcastHashJoin +(54) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(53) Exchange +(55) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(56) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(56) Exchange +(58) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(59) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(58) Project +(60) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(59) HashAggregate +(61) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(60) Exchange +(62) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(63) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(62) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(63) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(66) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(65) Exchange +(67) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) Sort +(68) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(67) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt index 04753848cb48..fc17c87d7df0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt @@ -1,12 +1,12 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- ^ ProjectExecTransformer (37) - +- ^ RegularHashAggregateExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ ProjectExecTransformer (34) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (33) + VeloxColumnarToRowExec (40) + +- ^ ProjectExecTransformer (38) + +- ^ RegularHashAggregateExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ ProjectExecTransformer (35) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (34) :- ^ ProjectExecTransformer (20) : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) : :- ^ InputIteratorTransformer (9) @@ -23,36 +23,37 @@ AdaptiveSparkPlan (59) : +- ^ ProjectExecTransformer (12) : +- ^ FilterExecTransformer (11) : +- ^ Scan parquet (10) - +- ^ FilterExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ FilterExecTransformer (22) - +- ^ Scan parquet (21) + +- ^ FilterExecTransformer (33) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ FilterExecTransformer (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (58) - +- HashAggregate (57) - +- Project (56) - +- ShuffledHashJoin Inner BuildRight (55) - :- Project (48) - : +- ShuffledHashJoin Inner BuildRight (47) - : :- Exchange (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Project (45) - : +- Filter (44) - : +- Scan parquet (43) - +- Filter (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- Filter (50) - +- Scan parquet (49) + HashAggregate (59) + +- HashAggregate (58) + +- Project (57) + +- ShuffledHashJoin Inner BuildRight (56) + :- Project (49) + : +- ShuffledHashJoin Inner BuildRight (48) + : :- Exchange (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Exchange (47) + : +- Project (46) + : +- Filter (45) + : +- Scan parquet (44) + +- Filter (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- Filter (51) + +- Scan parquet (50) (1) Scan parquet @@ -164,168 +165,172 @@ Input [3]: [l_partkey#X, sum#X, count#X] Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(32) FilterExecTransformer +(33) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(33) ShuffledHashJoinExecTransformer +(34) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(34) ProjectExecTransformer +(35) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(35) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(36) RegularHashAggregateExecTransformer +(37) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(37) ProjectExecTransformer +(38) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(38) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(39) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(40) Scan parquet +(41) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(41) Filter +(42) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(42) Exchange +(43) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Scan parquet +(44) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(45) Project +(46) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(46) Exchange +(47) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(48) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(48) Project +(49) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(49) Scan parquet +(50) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(50) Filter +(51) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(51) HashAggregate +(52) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(52) Exchange +(53) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(54) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(54) Filter +(55) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(55) ShuffledHashJoin +(56) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(56) Project +(57) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(57) HashAggregate +(58) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(58) HashAggregate +(59) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] -(59) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt index 516601a5f469..fc65f4b52897 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt @@ -1,18 +1,18 @@ == Physical Plan == -AdaptiveSparkPlan (102) +AdaptiveSparkPlan (103) +- == Final Plan == - VeloxColumnarToRowExec (69) - +- TakeOrderedAndProjectExecTransformer (68) - +- ^ RegularHashAggregateExecTransformer (66) - +- ^ RegularHashAggregateExecTransformer (65) - +- ^ ProjectExecTransformer (64) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (63) - :- ^ InputIteratorTransformer (45) - : +- ShuffleQueryStage (43) - : +- ColumnarExchange (42) - : +- VeloxAppendBatches (41) - : +- ^ ProjectExecTransformer (39) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (38) + VeloxColumnarToRowExec (70) + +- TakeOrderedAndProjectExecTransformer (69) + +- ^ RegularHashAggregateExecTransformer (67) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ ProjectExecTransformer (65) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (64) + :- ^ InputIteratorTransformer (46) + : +- ShuffleQueryStage (44) + : +- ColumnarExchange (43) + : +- VeloxAppendBatches (42) + : +- ^ ProjectExecTransformer (40) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (39) : :- ^ InputIteratorTransformer (9) : : +- ShuffleQueryStage (7) : : +- ColumnarExchange (6) @@ -20,12 +20,12 @@ AdaptiveSparkPlan (102) : : +- ^ ProjectExecTransformer (3) : : +- ^ FilterExecTransformer (2) : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (37) - : +- ShuffleQueryStage (35) - : +- ColumnarExchange (34) - : +- VeloxAppendBatches (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (30) + : +- ^ InputIteratorTransformer (38) + : +- ShuffleQueryStage (36) + : +- ColumnarExchange (35) + : +- VeloxAppendBatches (34) + : +- ^ ProjectExecTransformer (32) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (31) : :- ^ InputIteratorTransformer (18) : : +- ShuffleQueryStage (16) : : +- ColumnarExchange (15) @@ -33,62 +33,63 @@ AdaptiveSparkPlan (102) : : +- ^ ProjectExecTransformer (12) : : +- ^ FilterExecTransformer (11) : : +- ^ Scan parquet (10) - : +- ^ ProjectExecTransformer (29) - : +- ^ FilterExecTransformer (28) - : +- ^ RegularHashAggregateExecTransformer (27) - : +- ^ InputIteratorTransformer (26) - : +- ShuffleQueryStage (24) - : +- ColumnarExchange (23) - : +- ^ ProjectExecTransformer (21) - : +- ^ FlushableHashAggregateExecTransformer (20) - : +- ^ Scan parquet (19) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - :- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52) - : +- ColumnarExchange (51) - : +- VeloxAppendBatches (50) - : +- ^ ProjectExecTransformer (48) - : +- ^ FilterExecTransformer (47) - : +- ^ Scan parquet (46) - +- ^ ProjectExecTransformer (61) - +- ^ FilterExecTransformer (60) - +- ^ RegularHashAggregateExecTransformer (59) - +- ^ InputIteratorTransformer (58) - +- ShuffleQueryStage (56) - +- ReusedExchange (55) + : +- ^ ProjectExecTransformer (30) + : +- ^ FilterExecTransformer (29) + : +- ^ RegularHashAggregateExecTransformer (28) + : +- ^ InputIteratorTransformer (27) + : +- ShuffleQueryStage (25) + : +- ColumnarExchange (24) + : +- VeloxAppendBatches (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (63) + :- ^ InputIteratorTransformer (55) + : +- ShuffleQueryStage (53) + : +- ColumnarExchange (52) + : +- VeloxAppendBatches (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ Scan parquet (47) + +- ^ ProjectExecTransformer (62) + +- ^ FilterExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57) + +- ReusedExchange (56) +- == Initial Plan == - TakeOrderedAndProject (101) - +- HashAggregate (100) - +- HashAggregate (99) - +- Project (98) - +- ShuffledHashJoin Inner BuildRight (97) - :- Exchange (86) - : +- Project (85) - : +- ShuffledHashJoin Inner BuildLeft (84) - : :- Exchange (72) - : : +- Filter (71) - : : +- Scan parquet (70) - : +- Exchange (83) - : +- ShuffledHashJoin LeftSemi BuildRight (82) - : :- Exchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- Project (81) - : +- Filter (80) - : +- HashAggregate (79) - : +- Exchange (78) - : +- HashAggregate (77) - : +- Scan parquet (76) - +- ShuffledHashJoin LeftSemi BuildRight (96) - :- Exchange (89) - : +- Filter (88) - : +- Scan parquet (87) - +- Project (95) - +- Filter (94) - +- HashAggregate (93) - +- Exchange (92) - +- HashAggregate (91) - +- Scan parquet (90) + TakeOrderedAndProject (102) + +- HashAggregate (101) + +- HashAggregate (100) + +- Project (99) + +- ShuffledHashJoin Inner BuildRight (98) + :- Exchange (87) + : +- Project (86) + : +- ShuffledHashJoin Inner BuildLeft (85) + : :- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (84) + : +- ShuffledHashJoin LeftSemi BuildRight (83) + : :- Exchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Project (82) + : +- Filter (81) + : +- HashAggregate (80) + : +- Exchange (79) + : +- HashAggregate (78) + : +- Scan parquet (77) + +- ShuffledHashJoin LeftSemi BuildRight (97) + :- Exchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- Project (96) + +- Filter (95) + +- HashAggregate (94) + +- Exchange (93) + +- HashAggregate (92) + +- Scan parquet (91) (1) Scan parquet @@ -186,364 +187,368 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(28) FilterExecTransformer +(29) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(29) ProjectExecTransformer +(30) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(30) ShuffledHashJoinExecTransformer +(31) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(33) VeloxAppendBatches +(34) VeloxAppendBatches Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(34) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(35) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(36) InputAdapter +(37) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(38) ShuffledHashJoinExecTransformer +(39) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(39) ProjectExecTransformer +(40) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(40) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(41) VeloxAppendBatches +(42) VeloxAppendBatches Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(42) ColumnarExchange +(43) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(43) ShuffleQueryStage +(44) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(44) InputAdapter +(45) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(45) InputIteratorTransformer +(46) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(46) Scan parquet +(47) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(47) FilterExecTransformer +(48) FilterExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Arguments: isnotnull(l_orderkey#X) -(48) ProjectExecTransformer +(49) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(49) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(50) VeloxAppendBatches +(51) VeloxAppendBatches Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: X -(51) ColumnarExchange +(52) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(53) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(53) InputAdapter +(54) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(54) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(55) ReusedExchange [Reuses operator id: 23] +(56) ReusedExchange [Reuses operator id: 24] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(56) ShuffleQueryStage +(57) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(57) InputAdapter +(58) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(58) InputIteratorTransformer +(59) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(59) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(60) FilterExecTransformer +(61) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(61) ProjectExecTransformer +(62) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(62) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(63) ShuffledHashJoinExecTransformer +(64) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(64) ProjectExecTransformer +(65) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(65) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(66) RegularHashAggregateExecTransformer +(67) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(67) WholeStageCodegenTransformer (X) +(68) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(68) TakeOrderedAndProjectExecTransformer +(69) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(69) VeloxColumnarToRowExec +(70) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(70) Scan parquet +(71) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(72) Exchange +(73) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(73) Scan parquet +(74) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(75) Exchange +(76) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) Scan parquet +(77) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(77) HashAggregate +(78) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(78) Exchange +(79) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) HashAggregate +(80) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(80) Filter +(81) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(81) Project +(82) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(82) ShuffledHashJoin +(83) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(83) Exchange +(84) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) ShuffledHashJoin +(85) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(85) Project +(86) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(86) Exchange +(87) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Scan parquet +(88) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(88) Filter +(89) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(89) Exchange +(90) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(91) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(91) HashAggregate +(92) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(92) Exchange +(93) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) HashAggregate +(94) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(94) Filter +(95) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(95) Project +(96) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(96) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(97) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(98) Project +(99) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(99) HashAggregate +(100) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(100) HashAggregate +(101) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(101) TakeOrderedAndProject +(102) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(102) AdaptiveSparkPlan +(103) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt index 664bb4e88905..c0f3602f9fe6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt @@ -1,113 +1,114 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (129) +- == Final Plan == - VeloxColumnarToRowExec (91) - +- TakeOrderedAndProjectExecTransformer (90) - +- ^ RegularHashAggregateExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85) - +- ColumnarExchange (84) - +- ^ ProjectExecTransformer (82) - +- ^ FlushableHashAggregateExecTransformer (81) - +- ^ ProjectExecTransformer (80) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) - :- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) - : : :- ^ InputIteratorTransformer (9) - : : : +- ShuffleQueryStage (7) - : : : +- ColumnarExchange (6) - : : : +- VeloxAppendBatches (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (44) - : : +- ShuffleQueryStage (42) - : : +- ColumnarExchange (41) - : : +- VeloxAppendBatches (40) - : : +- ^ ProjectExecTransformer (38) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) - : : : :- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ FilterExecTransformer (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (26) - : : : +- ShuffleQueryStage (24) - : : : +- ColumnarExchange (23) - : : : +- VeloxAppendBatches (22) - : : : +- ^ ProjectExecTransformer (20) - : : : +- ^ Scan parquet (19) - : : +- ^ InputIteratorTransformer (36) - : : +- ShuffleQueryStage (34) - : : +- ColumnarExchange (33) - : : +- VeloxAppendBatches (32) - : : +- ^ ProjectExecTransformer (30) - : : +- ^ FilterExecTransformer (29) - : : +- ^ Scan parquet (28) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- VeloxAppendBatches (57) - : +- ^ ProjectExecTransformer (55) - : +- ^ FilterExecTransformer (54) - : +- ^ Scan parquet (53) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76) - +- ColumnarExchange (75) - +- VeloxAppendBatches (74) - +- ^ ProjectExecTransformer (72) - +- ^ FilterExecTransformer (71) - +- ^ Scan parquet (70) + VeloxColumnarToRowExec (92) + +- TakeOrderedAndProjectExecTransformer (91) + +- ^ RegularHashAggregateExecTransformer (89) + +- ^ InputIteratorTransformer (88) + +- ShuffleQueryStage (86) + +- ColumnarExchange (85) + +- VeloxAppendBatches (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ FilterExecTransformer (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ FilterExecTransformer (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ FilterExecTransformer (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- ShuffledHashJoin Inner BuildRight (122) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (110) - : : +- Project (109) - : : +- ShuffledHashJoin Inner BuildLeft (108) - : : :- Exchange (94) - : : : +- Filter (93) - : : : +- Scan parquet (92) - : : +- Exchange (107) - : : +- ShuffledHashJoin LeftAnti BuildRight (106) - : : :- ShuffledHashJoin LeftSemi BuildRight (101) - : : : :- Exchange (98) - : : : : +- Project (97) - : : : : +- Filter (96) - : : : : +- Scan parquet (95) - : : : +- Exchange (100) - : : : +- Scan parquet (99) - : : +- Exchange (105) - : : +- Project (104) - : : +- Filter (103) - : : +- Scan parquet (102) - : +- Exchange (114) - : +- Project (113) - : +- Filter (112) - : +- Scan parquet (111) - +- Exchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + TakeOrderedAndProject (128) + +- HashAggregate (127) + +- Exchange (126) + +- HashAggregate (125) + +- Project (124) + +- ShuffledHashJoin Inner BuildRight (123) + :- Exchange (118) + : +- Project (117) + : +- ShuffledHashJoin Inner BuildRight (116) + : :- Exchange (111) + : : +- Project (110) + : : +- ShuffledHashJoin Inner BuildLeft (109) + : : :- Exchange (95) + : : : +- Filter (94) + : : : +- Scan parquet (93) + : : +- Exchange (108) + : : +- ShuffledHashJoin LeftAnti BuildRight (107) + : : :- ShuffledHashJoin LeftSemi BuildRight (102) + : : : :- Exchange (99) + : : : : +- Project (98) + : : : : +- Filter (97) + : : : : +- Scan parquet (96) + : : : +- Exchange (101) + : : : +- Scan parquet (100) + : : +- Exchange (106) + : : +- Project (105) + : : +- Filter (104) + : : +- Scan parquet (103) + : +- Exchange (115) + : +- Project (114) + : +- Filter (113) + : +- Scan parquet (112) + +- Exchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) (1) Scan parquet @@ -449,210 +450,214 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(84) ColumnarExchange +(84) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(85) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(86) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(86) InputAdapter +(87) InputAdapter Input [2]: [s_name#X, count#X] -(87) InputIteratorTransformer +(88) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(88) RegularHashAggregateExecTransformer +(89) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(89) WholeStageCodegenTransformer (X) +(90) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(90) TakeOrderedAndProjectExecTransformer +(91) TakeOrderedAndProjectExecTransformer Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X], 0 -(91) VeloxColumnarToRowExec +(92) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(92) Scan parquet +(93) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(93) Filter +(94) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(94) Exchange +(95) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Scan parquet +(96) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(96) Filter +(97) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(97) Project +(98) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(98) Exchange +(99) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(100) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(100) Exchange +(101) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) ShuffledHashJoin +(102) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(102) Scan parquet +(103) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(103) Filter +(104) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(104) Project +(105) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(105) Exchange +(106) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(107) Exchange +(108) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(109) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(109) Project +(110) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(110) Exchange +(111) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(112) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(112) Filter +(113) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(113) Project +(114) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(114) Exchange +(115) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(116) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(116) Project +(117) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(117) Exchange +(118) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(119) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(119) Filter +(120) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(120) Project +(121) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(121) Exchange +(122) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) ShuffledHashJoin +(123) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(123) Project +(124) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(124) HashAggregate +(125) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(125) Exchange +(126) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(127) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(127) TakeOrderedAndProject +(128) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(128) AdaptiveSparkPlan +(129) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt index 5ada352ba19c..5ab0811e658f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt @@ -1,46 +1,47 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (50) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- VeloxAppendBatches (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ ProjectExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin LeftAnti BuildRight (42) - :- Exchange (39) - : +- Filter (38) - : +- Scan parquet (37) - +- Exchange (41) - +- Scan parquet (40) + Sort (49) + +- Exchange (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftAnti BuildRight (43) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (42) + +- Scan parquet (41) (1) Scan parquet @@ -136,120 +137,124 @@ Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(29) VeloxAppendBatches +(30) VeloxAppendBatches Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(30) ColumnarExchange +(31) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(32) InputAdapter +(33) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(33) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) SortExecTransformer +(35) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(35) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(36) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(37) Scan parquet +(38) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(38) Filter +(39) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(39) Exchange +(40) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) Scan parquet +(41) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(41) Exchange +(42) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(44) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(45) Exchange +(46) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(47) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(47) Exchange +(48) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(49) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(50) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt index 2f1dd53da1ef..1b95ae3dbf39 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt @@ -1,50 +1,51 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32) - +- ColumnarExchange (31) - +- VeloxAppendBatches (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- VeloxAppendBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ FilterExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (52) - +- Exchange (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- ShuffledHashJoin LeftSemi BuildRight (46) - :- Exchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (53) + +- Exchange (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftSemi BuildRight (47) + :- Exchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -145,133 +146,137 @@ Input [2]: [o_orderpriority#X, count#X] Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(24) ColumnarExchange +(24) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(28) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(29) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(30) VeloxAppendBatches +(31) VeloxAppendBatches Input [2]: [o_orderpriority#X, order_count#X] Arguments: X -(31) ColumnarExchange +(32) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(35) SortExecTransformer +(36) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(37) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(41) Exchange +(42) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Scan parquet +(43) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(43) Filter +(44) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(44) Project +(45) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(45) Exchange +(46) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(47) Project +(48) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(48) HashAggregate +(49) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(49) Exchange +(50) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(51) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(51) Exchange +(52) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(53) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt index e20b31dd7907..c31fbccc1e59 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt @@ -1,126 +1,127 @@ == Physical Plan == -AdaptiveSparkPlan (145) +AdaptiveSparkPlan (146) +- == Final Plan == - VeloxColumnarToRowExec (105) - +- ^ SortExecTransformer (103) - +- ^ InputIteratorTransformer (102) - +- ShuffleQueryStage (100) - +- ColumnarExchange (99) - +- VeloxAppendBatches (98) - +- ^ RegularHashAggregateExecTransformer (96) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93) - +- ColumnarExchange (92) - +- ^ ProjectExecTransformer (90) - +- ^ FlushableHashAggregateExecTransformer (89) - +- ^ ProjectExecTransformer (88) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - :- ^ InputIteratorTransformer (77) - : +- ShuffleQueryStage (75) - : +- ColumnarExchange (74) - : +- VeloxAppendBatches (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - : :- ^ InputIteratorTransformer (60) - : : +- ShuffleQueryStage (58) - : : +- ColumnarExchange (57) - : : +- VeloxAppendBatches (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - : : :- ^ InputIteratorTransformer (43) - : : : +- ShuffleQueryStage (41) - : : : +- ColumnarExchange (40) - : : : +- VeloxAppendBatches (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - : : : :- ^ InputIteratorTransformer (26) - : : : : +- ShuffleQueryStage (24) - : : : : +- ColumnarExchange (23) - : : : : +- VeloxAppendBatches (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ InputIteratorTransformer (9) - : : : : : +- ShuffleQueryStage (7) - : : : : : +- ColumnarExchange (6) - : : : : : +- VeloxAppendBatches (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ FilterExecTransformer (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (35) - : : : +- ShuffleQueryStage (33) - : : : +- ColumnarExchange (32) - : : : +- VeloxAppendBatches (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ FilterExecTransformer (28) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ FilterExecTransformer (45) - : : +- ^ Scan parquet (44) - : +- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ FilterExecTransformer (62) - : +- ^ Scan parquet (61) - +- ^ InputIteratorTransformer (86) - +- ShuffleQueryStage (84) - +- ColumnarExchange (83) - +- VeloxAppendBatches (82) - +- ^ ProjectExecTransformer (80) - +- ^ FilterExecTransformer (79) - +- ^ Scan parquet (78) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (144) - +- Exchange (143) - +- HashAggregate (142) - +- Exchange (141) - +- HashAggregate (140) - +- Project (139) - +- ShuffledHashJoin Inner BuildRight (138) - :- Exchange (133) - : +- Project (132) - : +- ShuffledHashJoin Inner BuildRight (131) - : :- Exchange (127) - : : +- Project (126) - : : +- ShuffledHashJoin Inner BuildRight (125) - : : :- Exchange (121) - : : : +- Project (120) - : : : +- ShuffledHashJoin Inner BuildRight (119) - : : : :- Exchange (115) - : : : : +- Project (114) - : : : : +- ShuffledHashJoin Inner BuildLeft (113) - : : : : :- Exchange (108) - : : : : : +- Filter (107) - : : : : : +- Scan parquet (106) - : : : : +- Exchange (112) - : : : : +- Project (111) - : : : : +- Filter (110) - : : : : +- Scan parquet (109) - : : : +- Exchange (118) - : : : +- Filter (117) - : : : +- Scan parquet (116) - : : +- Exchange (124) - : : +- Filter (123) - : : +- Scan parquet (122) - : +- Exchange (130) - : +- Filter (129) - : +- Scan parquet (128) - +- Exchange (137) - +- Project (136) - +- Filter (135) - +- Scan parquet (134) + Sort (145) + +- Exchange (144) + +- HashAggregate (143) + +- Exchange (142) + +- HashAggregate (141) + +- Project (140) + +- ShuffledHashJoin Inner BuildRight (139) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Project (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (138) + +- Project (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -493,245 +494,249 @@ Input [3]: [n_name#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(92) ColumnarExchange +(92) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(93) ShuffleQueryStage +(94) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(94) InputAdapter +(95) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(95) InputIteratorTransformer +(96) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(96) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(97) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(98) VeloxAppendBatches +(99) VeloxAppendBatches Input [2]: [n_name#X, revenue#X] Arguments: X -(99) ColumnarExchange +(100) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(100) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(101) InputAdapter +(102) InputAdapter Input [2]: [n_name#X, revenue#X] -(102) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(103) SortExecTransformer +(104) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(104) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(105) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(106) Scan parquet +(107) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(108) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(109) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Scan parquet +(110) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(110) Filter +(111) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(111) Project +(112) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(112) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(114) Project +(115) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(115) Exchange +(116) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) Scan parquet +(117) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(117) Filter +(118) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(118) Exchange +(119) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(120) Project +(121) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(121) Exchange +(122) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(123) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(123) Filter +(124) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(124) Exchange +(125) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(126) Project +(127) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(127) Exchange +(128) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(129) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(129) Filter +(130) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(130) Exchange +(131) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(132) Project +(133) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(133) Exchange +(134) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(134) Scan parquet +(135) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(135) Filter +(136) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(136) Project +(137) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(137) Exchange +(138) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(138) ShuffledHashJoin +(139) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(139) Project +(140) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(140) HashAggregate +(141) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(141) Exchange +(142) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) HashAggregate +(143) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(143) Exchange +(144) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(144) Sort +(145) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(145) AdaptiveSparkPlan +(146) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/6.txt index 41f69e74d1a0..786a89fe715a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/6.txt @@ -7,7 +7,7 @@ AdaptiveSparkPlan (20) +- ShuffleQueryStage (8) +- ColumnarExchange (7) +- VeloxAppendBatches (6) - +- ^ RegularHashAggregateExecTransformer (4) + +- ^ FlushableHashAggregateExecTransformer (4) +- ^ ProjectExecTransformer (3) +- ^ FilterExecTransformer (2) +- ^ Scan parquet (1) @@ -35,7 +35,7 @@ Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnot Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) RegularHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt index e7c225796976..06b84fdca2c7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt @@ -1,120 +1,121 @@ == Physical Plan == -AdaptiveSparkPlan (138) +AdaptiveSparkPlan (139) +- == Final Plan == - VeloxColumnarToRowExec (100) - +- ^ SortExecTransformer (98) - +- ^ InputIteratorTransformer (97) - +- ShuffleQueryStage (95) - +- ColumnarExchange (94) - +- VeloxAppendBatches (93) - +- ^ RegularHashAggregateExecTransformer (91) - +- ^ InputIteratorTransformer (90) - +- ShuffleQueryStage (88) - +- ColumnarExchange (87) - +- ^ ProjectExecTransformer (85) - +- ^ FlushableHashAggregateExecTransformer (84) - +- ^ ProjectExecTransformer (83) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) - :- ^ InputIteratorTransformer (77) - : +- ShuffleQueryStage (75) - : +- ColumnarExchange (74) - : +- VeloxAppendBatches (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - : :- ^ InputIteratorTransformer (60) - : : +- ShuffleQueryStage (58) - : : +- ColumnarExchange (57) - : : +- VeloxAppendBatches (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - : : :- ^ InputIteratorTransformer (43) - : : : +- ShuffleQueryStage (41) - : : : +- ColumnarExchange (40) - : : : +- VeloxAppendBatches (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : :- ^ InputIteratorTransformer (26) - : : : : +- ShuffleQueryStage (24) - : : : : +- ColumnarExchange (23) - : : : : +- VeloxAppendBatches (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ InputIteratorTransformer (9) - : : : : : +- ShuffleQueryStage (7) - : : : : : +- ColumnarExchange (6) - : : : : : +- VeloxAppendBatches (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ FilterExecTransformer (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (35) - : : : +- ShuffleQueryStage (33) - : : : +- ColumnarExchange (32) - : : : +- VeloxAppendBatches (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ FilterExecTransformer (28) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ FilterExecTransformer (45) - : : +- ^ Scan parquet (44) - : +- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ FilterExecTransformer (62) - : +- ^ Scan parquet (61) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79) - +- ReusedExchange (78) + VeloxColumnarToRowExec (101) + +- ^ SortExecTransformer (99) + +- ^ InputIteratorTransformer (98) + +- ShuffleQueryStage (96) + +- ColumnarExchange (95) + +- VeloxAppendBatches (94) + +- ^ RegularHashAggregateExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ShuffleQueryStage (89) + +- ColumnarExchange (88) + +- VeloxAppendBatches (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79) + +- ReusedExchange (78) +- == Initial Plan == - Sort (137) - +- Exchange (136) - +- HashAggregate (135) - +- Exchange (134) - +- HashAggregate (133) - +- Project (132) - +- ShuffledHashJoin Inner BuildRight (131) - :- Exchange (127) - : +- Project (126) - : +- ShuffledHashJoin Inner BuildRight (125) - : :- Exchange (121) - : : +- Project (120) - : : +- ShuffledHashJoin Inner BuildRight (119) - : : :- Exchange (115) - : : : +- Project (114) - : : : +- ShuffledHashJoin Inner BuildRight (113) - : : : :- Exchange (109) - : : : : +- Project (108) - : : : : +- ShuffledHashJoin Inner BuildLeft (107) - : : : : :- Exchange (103) - : : : : : +- Filter (102) - : : : : : +- Scan parquet (101) - : : : : +- Exchange (106) - : : : : +- Filter (105) - : : : : +- Scan parquet (104) - : : : +- Exchange (112) - : : : +- Filter (111) - : : : +- Scan parquet (110) - : : +- Exchange (118) - : : +- Filter (117) - : : +- Scan parquet (116) - : +- Exchange (124) - : +- Filter (123) - : +- Scan parquet (122) - +- Exchange (130) - +- Filter (129) - +- Scan parquet (128) + Sort (138) + +- Exchange (137) + +- HashAggregate (136) + +- Exchange (135) + +- HashAggregate (134) + +- Project (133) + +- ShuffledHashJoin Inner BuildRight (132) + :- Exchange (128) + : +- Project (127) + : +- ShuffledHashJoin Inner BuildRight (126) + : :- Exchange (122) + : : +- Project (121) + : : +- ShuffledHashJoin Inner BuildRight (120) + : : :- Exchange (116) + : : : +- Project (115) + : : : +- ShuffledHashJoin Inner BuildRight (114) + : : : :- Exchange (110) + : : : : +- Project (109) + : : : : +- ShuffledHashJoin Inner BuildLeft (108) + : : : : :- Exchange (104) + : : : : : +- Filter (103) + : : : : : +- Scan parquet (102) + : : : : +- Exchange (107) + : : : : +- Filter (106) + : : : : +- Scan parquet (105) + : : : +- Exchange (113) + : : : +- Filter (112) + : : : +- Scan parquet (111) + : : +- Exchange (119) + : : +- Filter (118) + : : +- Scan parquet (117) + : +- Exchange (125) + : +- Filter (124) + : +- Scan parquet (123) + +- Exchange (131) + +- Filter (130) + +- Scan parquet (129) (1) Scan parquet @@ -463,237 +464,241 @@ Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(87) ColumnarExchange +(87) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(88) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(88) ShuffleQueryStage +(89) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(89) InputAdapter +(90) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(90) InputIteratorTransformer +(91) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(91) RegularHashAggregateExecTransformer +(92) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(92) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(93) VeloxAppendBatches +(94) VeloxAppendBatches Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(94) ColumnarExchange +(95) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(95) ShuffleQueryStage +(96) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(96) InputAdapter +(97) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(97) InputIteratorTransformer +(98) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(98) SortExecTransformer +(99) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(99) WholeStageCodegenTransformer (X) +(100) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(100) VeloxColumnarToRowExec +(101) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(101) Scan parquet +(102) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(102) Filter +(103) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(103) Exchange +(104) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) Scan parquet +(105) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(105) Filter +(106) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(106) Exchange +(107) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(108) Project +(109) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(109) Exchange +(110) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(112) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(114) Project +(115) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(115) Exchange +(116) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) Scan parquet +(117) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(117) Filter +(118) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(118) Exchange +(119) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(120) Project +(121) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(121) Exchange +(122) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(123) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(123) Filter +(124) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(124) Exchange +(125) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(126) Project +(127) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(127) Exchange +(128) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(129) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(129) Filter +(130) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(130) Exchange +(131) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(132) Project +(133) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(133) HashAggregate +(134) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(134) Exchange +(135) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(135) HashAggregate +(136) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(136) Exchange +(137) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) Sort +(138) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(138) AdaptiveSparkPlan +(139) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt index de63f62ffc7e..e9fdc420f128 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt @@ -1,165 +1,166 @@ == Physical Plan == -AdaptiveSparkPlan (192) +AdaptiveSparkPlan (193) +- == Final Plan == - VeloxColumnarToRowExec (140) - +- ^ SortExecTransformer (138) - +- ^ InputIteratorTransformer (137) - +- ShuffleQueryStage (135) - +- ColumnarExchange (134) - +- VeloxAppendBatches (133) - +- ^ ProjectExecTransformer (131) - +- ^ RegularHashAggregateExecTransformer (130) - +- ^ InputIteratorTransformer (129) - +- ShuffleQueryStage (127) - +- ColumnarExchange (126) - +- ^ ProjectExecTransformer (124) - +- ^ FlushableHashAggregateExecTransformer (123) - +- ^ ProjectExecTransformer (122) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) - :- ^ InputIteratorTransformer (111) - : +- ShuffleQueryStage (109) - : +- ColumnarExchange (108) - : +- VeloxAppendBatches (107) - : +- ^ ProjectExecTransformer (105) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) - : :- ^ InputIteratorTransformer (94) - : : +- ShuffleQueryStage (92) - : : +- ColumnarExchange (91) - : : +- VeloxAppendBatches (90) - : : +- ^ ProjectExecTransformer (88) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - : : :- ^ InputIteratorTransformer (77) - : : : +- ShuffleQueryStage (75) - : : : +- ColumnarExchange (74) - : : : +- VeloxAppendBatches (73) - : : : +- ^ ProjectExecTransformer (71) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) - : : : :- ^ InputIteratorTransformer (60) - : : : : +- ShuffleQueryStage (58) - : : : : +- ColumnarExchange (57) - : : : : +- VeloxAppendBatches (56) - : : : : +- ^ ProjectExecTransformer (54) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) - : : : : :- ^ InputIteratorTransformer (43) - : : : : : +- ShuffleQueryStage (41) - : : : : : +- ColumnarExchange (40) - : : : : : +- VeloxAppendBatches (39) - : : : : : +- ^ ProjectExecTransformer (37) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : : : :- ^ InputIteratorTransformer (26) - : : : : : : +- ShuffleQueryStage (24) - : : : : : : +- ColumnarExchange (23) - : : : : : : +- VeloxAppendBatches (22) - : : : : : : +- ^ ProjectExecTransformer (20) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : : : :- ^ InputIteratorTransformer (9) - : : : : : : : +- ShuffleQueryStage (7) - : : : : : : : +- ColumnarExchange (6) - : : : : : : : +- VeloxAppendBatches (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (18) - : : : : : : +- ShuffleQueryStage (16) - : : : : : : +- ColumnarExchange (15) - : : : : : : +- VeloxAppendBatches (14) - : : : : : : +- ^ ProjectExecTransformer (12) - : : : : : : +- ^ FilterExecTransformer (11) - : : : : : : +- ^ Scan parquet (10) - : : : : : +- ^ InputIteratorTransformer (35) - : : : : : +- ShuffleQueryStage (33) - : : : : : +- ColumnarExchange (32) - : : : : : +- VeloxAppendBatches (31) - : : : : : +- ^ ProjectExecTransformer (29) - : : : : : +- ^ FilterExecTransformer (28) - : : : : : +- ^ Scan parquet (27) - : : : : +- ^ InputIteratorTransformer (52) - : : : : +- ShuffleQueryStage (50) - : : : : +- ColumnarExchange (49) - : : : : +- VeloxAppendBatches (48) - : : : : +- ^ ProjectExecTransformer (46) - : : : : +- ^ FilterExecTransformer (45) - : : : : +- ^ Scan parquet (44) - : : : +- ^ InputIteratorTransformer (69) - : : : +- ShuffleQueryStage (67) - : : : +- ColumnarExchange (66) - : : : +- VeloxAppendBatches (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ FilterExecTransformer (62) - : : : +- ^ Scan parquet (61) - : : +- ^ InputIteratorTransformer (86) - : : +- ShuffleQueryStage (84) - : : +- ColumnarExchange (83) - : : +- VeloxAppendBatches (82) - : : +- ^ ProjectExecTransformer (80) - : : +- ^ FilterExecTransformer (79) - : : +- ^ Scan parquet (78) - : +- ^ InputIteratorTransformer (103) - : +- ShuffleQueryStage (101) - : +- ColumnarExchange (100) - : +- VeloxAppendBatches (99) - : +- ^ ProjectExecTransformer (97) - : +- ^ FilterExecTransformer (96) - : +- ^ Scan parquet (95) - +- ^ InputIteratorTransformer (120) - +- ShuffleQueryStage (118) - +- ColumnarExchange (117) - +- VeloxAppendBatches (116) - +- ^ ProjectExecTransformer (114) - +- ^ FilterExecTransformer (113) - +- ^ Scan parquet (112) + VeloxColumnarToRowExec (141) + +- ^ SortExecTransformer (139) + +- ^ InputIteratorTransformer (138) + +- ShuffleQueryStage (136) + +- ColumnarExchange (135) + +- VeloxAppendBatches (134) + +- ^ ProjectExecTransformer (132) + +- ^ RegularHashAggregateExecTransformer (131) + +- ^ InputIteratorTransformer (130) + +- ShuffleQueryStage (128) + +- ColumnarExchange (127) + +- VeloxAppendBatches (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ FilterExecTransformer (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ FilterExecTransformer (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ FilterExecTransformer (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ FilterExecTransformer (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ FilterExecTransformer (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ FilterExecTransformer (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ FilterExecTransformer (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (191) - +- Exchange (190) - +- HashAggregate (189) - +- Exchange (188) - +- HashAggregate (187) - +- Project (186) - +- ShuffledHashJoin Inner BuildRight (185) - :- Exchange (180) - : +- Project (179) - : +- ShuffledHashJoin Inner BuildRight (178) - : :- Exchange (174) - : : +- Project (173) - : : +- ShuffledHashJoin Inner BuildRight (172) - : : :- Exchange (168) - : : : +- Project (167) - : : : +- ShuffledHashJoin Inner BuildRight (166) - : : : :- Exchange (162) - : : : : +- Project (161) - : : : : +- ShuffledHashJoin Inner BuildRight (160) - : : : : :- Exchange (156) - : : : : : +- Project (155) - : : : : : +- ShuffledHashJoin Inner BuildRight (154) - : : : : : :- Exchange (150) - : : : : : : +- Project (149) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (148) - : : : : : : :- Exchange (144) - : : : : : : : +- Project (143) - : : : : : : : +- Filter (142) - : : : : : : : +- Scan parquet (141) - : : : : : : +- Exchange (147) - : : : : : : +- Filter (146) - : : : : : : +- Scan parquet (145) - : : : : : +- Exchange (153) - : : : : : +- Filter (152) - : : : : : +- Scan parquet (151) - : : : : +- Exchange (159) - : : : : +- Filter (158) - : : : : +- Scan parquet (157) - : : : +- Exchange (165) - : : : +- Filter (164) - : : : +- Scan parquet (163) - : : +- Exchange (171) - : : +- Filter (170) - : : +- Scan parquet (169) - : +- Exchange (177) - : +- Filter (176) - : +- Scan parquet (175) - +- Exchange (184) - +- Project (183) - +- Filter (182) - +- Scan parquet (181) + Sort (192) + +- Exchange (191) + +- HashAggregate (190) + +- Exchange (189) + +- HashAggregate (188) + +- Project (187) + +- ShuffledHashJoin Inner BuildRight (186) + :- Exchange (181) + : +- Project (180) + : +- ShuffledHashJoin Inner BuildRight (179) + : :- Exchange (175) + : : +- Project (174) + : : +- ShuffledHashJoin Inner BuildRight (173) + : : :- Exchange (169) + : : : +- Project (168) + : : : +- ShuffledHashJoin Inner BuildRight (167) + : : : :- Exchange (163) + : : : : +- Project (162) + : : : : +- ShuffledHashJoin Inner BuildRight (161) + : : : : :- Exchange (157) + : : : : : +- Project (156) + : : : : : +- ShuffledHashJoin Inner BuildRight (155) + : : : : : :- Exchange (151) + : : : : : : +- Project (150) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (149) + : : : : : : :- Exchange (145) + : : : : : : : +- Project (144) + : : : : : : : +- Filter (143) + : : : : : : : +- Scan parquet (142) + : : : : : : +- Exchange (148) + : : : : : : +- Filter (147) + : : : : : : +- Scan parquet (146) + : : : : : +- Exchange (154) + : : : : : +- Filter (153) + : : : : : +- Scan parquet (152) + : : : : +- Exchange (160) + : : : : +- Filter (159) + : : : : +- Scan parquet (158) + : : : +- Exchange (166) + : : : +- Filter (165) + : : : +- Scan parquet (164) + : : +- Exchange (172) + : : +- Filter (171) + : : +- Scan parquet (170) + : +- Exchange (178) + : +- Filter (177) + : +- Scan parquet (176) + +- Exchange (185) + +- Project (184) + +- Filter (183) + +- Scan parquet (182) (1) Scan parquet @@ -668,305 +669,309 @@ Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(126) ColumnarExchange +(126) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(127) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(127) ShuffleQueryStage +(128) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(128) InputAdapter +(129) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(129) InputIteratorTransformer +(130) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(130) RegularHashAggregateExecTransformer +(131) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(131) ProjectExecTransformer +(132) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(132) WholeStageCodegenTransformer (X) +(133) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(133) VeloxAppendBatches +(134) VeloxAppendBatches Input [2]: [o_year#X, mkt_share#X] Arguments: X -(134) ColumnarExchange +(135) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(135) ShuffleQueryStage +(136) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(136) InputAdapter +(137) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(137) InputIteratorTransformer +(138) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(138) SortExecTransformer +(139) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(139) WholeStageCodegenTransformer (X) +(140) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(140) VeloxColumnarToRowExec +(141) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(141) Scan parquet +(142) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(142) Filter +(143) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(143) Project +(144) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(144) Exchange +(145) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) Scan parquet +(146) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(146) Filter +(147) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(147) Exchange +(148) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) ShuffledHashJoin +(149) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(149) Project +(150) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(150) Exchange +(151) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) Scan parquet +(152) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(152) Filter +(153) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(153) Exchange +(154) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) ShuffledHashJoin +(155) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(155) Project +(156) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(156) Exchange +(157) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) Scan parquet +(158) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(158) Filter +(159) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(159) Exchange +(160) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) ShuffledHashJoin +(161) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(161) Project +(162) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(162) Exchange +(163) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) Scan parquet +(164) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(164) Filter +(165) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(165) Exchange +(166) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) ShuffledHashJoin +(167) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(167) Project +(168) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(168) Exchange +(169) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(169) Scan parquet +(170) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(170) Filter +(171) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(171) Exchange +(172) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(172) ShuffledHashJoin +(173) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(173) Project +(174) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(174) Exchange +(175) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(175) Scan parquet +(176) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(176) Filter +(177) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(177) Exchange +(178) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(178) ShuffledHashJoin +(179) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(179) Project +(180) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(180) Exchange +(181) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(181) Scan parquet +(182) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(182) Filter +(183) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(183) Project +(184) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(184) Exchange +(185) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(185) ShuffledHashJoin +(186) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(186) Project +(187) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(187) HashAggregate +(188) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(188) Exchange +(189) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(189) HashAggregate +(190) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(190) Exchange +(191) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(191) Sort +(192) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(192) AdaptiveSparkPlan +(193) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt index 1fc81ef08365..d6bc308a9c2a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt @@ -1,125 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (144) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (105) - +- ^ SortExecTransformer (103) - +- ^ InputIteratorTransformer (102) - +- ShuffleQueryStage (100) - +- ColumnarExchange (99) - +- VeloxAppendBatches (98) - +- ^ RegularHashAggregateExecTransformer (96) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93) - +- ColumnarExchange (92) - +- ^ ProjectExecTransformer (90) - +- ^ FlushableHashAggregateExecTransformer (89) - +- ^ ProjectExecTransformer (88) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - :- ^ InputIteratorTransformer (77) - : +- ShuffleQueryStage (75) - : +- ColumnarExchange (74) - : +- VeloxAppendBatches (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) - : :- ^ InputIteratorTransformer (60) - : : +- ShuffleQueryStage (58) - : : +- ColumnarExchange (57) - : : +- VeloxAppendBatches (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) - : : :- ^ InputIteratorTransformer (43) - : : : +- ShuffleQueryStage (41) - : : : +- ColumnarExchange (40) - : : : +- VeloxAppendBatches (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : :- ^ InputIteratorTransformer (26) - : : : : +- ShuffleQueryStage (24) - : : : : +- ColumnarExchange (23) - : : : : +- VeloxAppendBatches (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ InputIteratorTransformer (9) - : : : : : +- ShuffleQueryStage (7) - : : : : : +- ColumnarExchange (6) - : : : : : +- VeloxAppendBatches (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ FilterExecTransformer (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (35) - : : : +- ShuffleQueryStage (33) - : : : +- ColumnarExchange (32) - : : : +- VeloxAppendBatches (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ FilterExecTransformer (28) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ FilterExecTransformer (45) - : : +- ^ Scan parquet (44) - : +- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ FilterExecTransformer (62) - : +- ^ Scan parquet (61) - +- ^ InputIteratorTransformer (86) - +- ShuffleQueryStage (84) - +- ColumnarExchange (83) - +- VeloxAppendBatches (82) - +- ^ ProjectExecTransformer (80) - +- ^ FilterExecTransformer (79) - +- ^ Scan parquet (78) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (143) - +- Exchange (142) - +- HashAggregate (141) - +- Exchange (140) - +- HashAggregate (139) - +- Project (138) - +- ShuffledHashJoin Inner BuildRight (137) - :- Exchange (133) - : +- Project (132) - : +- ShuffledHashJoin Inner BuildRight (131) - : :- Exchange (127) - : : +- Project (126) - : : +- ShuffledHashJoin Inner BuildRight (125) - : : :- Exchange (121) - : : : +- Project (120) - : : : +- ShuffledHashJoin Inner BuildRight (119) - : : : :- Exchange (115) - : : : : +- Project (114) - : : : : +- ShuffledHashJoin Inner BuildLeft (113) - : : : : :- Exchange (109) - : : : : : +- Project (108) - : : : : : +- Filter (107) - : : : : : +- Scan parquet (106) - : : : : +- Exchange (112) - : : : : +- Filter (111) - : : : : +- Scan parquet (110) - : : : +- Exchange (118) - : : : +- Filter (117) - : : : +- Scan parquet (116) - : : +- Exchange (124) - : : +- Filter (123) - : : +- Scan parquet (122) - : +- Exchange (130) - : +- Filter (129) - : +- Scan parquet (128) - +- Exchange (136) - +- Filter (135) - +- Scan parquet (134) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (110) + : : : : : +- Project (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Filter (112) + : : : : +- Scan parquet (111) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -492,241 +493,245 @@ Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(92) ColumnarExchange +(92) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(93) ShuffleQueryStage +(94) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(94) InputAdapter +(95) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(95) InputIteratorTransformer +(96) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(96) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(97) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(98) VeloxAppendBatches +(99) VeloxAppendBatches Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(99) ColumnarExchange +(100) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(100) ShuffleQueryStage +(101) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(101) InputAdapter +(102) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(102) InputIteratorTransformer +(103) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(103) SortExecTransformer +(104) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(104) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(105) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(106) Scan parquet +(107) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(107) Filter +(108) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(108) Project +(109) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(109) Exchange +(110) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(111) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(112) Exchange +(113) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(114) Project +(115) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(115) Exchange +(116) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) Scan parquet +(117) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(117) Filter +(118) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(118) Exchange +(119) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(120) Project +(121) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(121) Exchange +(122) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(123) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(123) Filter +(124) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(124) Exchange +(125) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(126) Project +(127) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(127) Exchange +(128) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(129) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(129) Filter +(130) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(130) Exchange +(131) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(132) Project +(133) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(133) Exchange +(134) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(134) Scan parquet +(135) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(135) Filter +(136) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(136) Exchange +(137) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(138) Project +(139) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(139) HashAggregate +(140) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(140) Exchange +(141) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(141) HashAggregate +(142) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(142) Exchange +(143) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(143) Sort +(144) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(144) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/1.txt index 2f3257bbdafc..22dd5100c4fb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (29) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (28) - +- Exchange (27) - +- HashAggregate (26) - +- Exchange (25) - +- HashAggregate (24) - +- Project (23) - +- Filter (22) - +- Scan parquet (21) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -57,101 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) VeloxAppendBatches +(14) VeloxAppendBatches Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) ColumnarExchange +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(18) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(20) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(21) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(22) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(23) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(24) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(25) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(27) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(29) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt index 8ca3c5d087a8..fcb13291c838 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt @@ -1,84 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (66) - +- TakeOrderedAndProjectExecTransformer (65) - +- ^ ProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ ProjectExecTransformer (56) - +- ^ FlushableHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - :- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41), Statistics(X) - : +- ColumnarExchange (40) - : +- VeloxAppendBatches (39) - : +- ^ ProjectExecTransformer (37) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - : :- ^ InputIteratorTransformer (26) - : : +- ShuffleQueryStage (24), Statistics(X) - : : +- ColumnarExchange (23) - : : +- VeloxAppendBatches (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : : :- ^ InputIteratorTransformer (9) - : : : +- ShuffleQueryStage (7), Statistics(X) - : : : +- ColumnarExchange (6) - : : : +- VeloxAppendBatches (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (18) - : : +- ShuffleQueryStage (16), Statistics(X) - : : +- ColumnarExchange (15) - : : +- VeloxAppendBatches (14) - : : +- ^ ProjectExecTransformer (12) - : : +- ^ FilterExecTransformer (11) - : : +- ^ Scan parquet (10) - : +- ^ InputIteratorTransformer (35) - : +- ShuffleQueryStage (33), Statistics(X) - : +- ColumnarExchange (32) - : +- VeloxAppendBatches (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ FilterExecTransformer (28) - : +- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- VeloxAppendBatches (48) - +- ^ ProjectExecTransformer (46) - +- ^ FilterExecTransformer (45) - +- ^ Scan parquet (44) + VeloxColumnarToRowExec (67) + +- TakeOrderedAndProjectExecTransformer (66) + +- ^ ProjectExecTransformer (64) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24), Statistics(X) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33), Statistics(X) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FilterExecTransformer (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (92) - +- HashAggregate (91) - +- Exchange (90) - +- HashAggregate (89) - +- Project (88) - +- ShuffledHashJoin Inner BuildRight (87) - :- Exchange (83) - : +- Project (82) - : +- ShuffledHashJoin Inner BuildRight (81) - : :- Exchange (76) - : : +- Project (75) - : : +- ShuffledHashJoin Inner BuildRight (74) - : : :- Exchange (69) - : : : +- Filter (68) - : : : +- Scan parquet (67) - : : +- Exchange (73) - : : +- Project (72) - : : +- Filter (71) - : : +- Scan parquet (70) - : +- Exchange (80) - : +- Project (79) - : +- Filter (78) - : +- Scan parquet (77) - +- Exchange (86) - +- Filter (85) - +- Scan parquet (84) + TakeOrderedAndProject (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- ShuffledHashJoin Inner BuildRight (88) + :- Exchange (84) + : +- Project (83) + : +- ShuffledHashJoin Inner BuildRight (82) + : :- Exchange (77) + : : +- Project (76) + : : +- ShuffledHashJoin Inner BuildRight (75) + : : :- Exchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- Exchange (74) + : : +- Project (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (81) + : +- Project (80) + : +- Filter (79) + : +- Scan parquet (78) + +- Exchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -315,167 +316,171 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(58) ColumnarExchange +(58) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(59) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(60) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(60) InputAdapter +(61) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(61) InputIteratorTransformer +(62) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(62) RegularHashAggregateExecTransformer +(63) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(63) ProjectExecTransformer +(64) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(65) TakeOrderedAndProjectExecTransformer +(66) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(66) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) Scan parquet +(68) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(68) Filter +(69) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(69) Exchange +(70) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(70) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(72) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(73) Exchange +(74) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) ShuffledHashJoin +(75) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(75) Project +(76) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(76) Exchange +(77) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Scan parquet +(78) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(78) Filter +(79) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(79) Project +(80) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(80) Exchange +(81) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(82) Project +(83) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(83) Exchange +(84) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(86) Exchange +(87) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) ShuffledHashJoin +(88) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) Project +(89) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(89) HashAggregate +(90) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(90) Exchange +(91) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(92) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(92) TakeOrderedAndProject +(93) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(93) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt index 0e370c4f114a..bbd20320b798 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt @@ -1,70 +1,71 @@ == Physical Plan == -AdaptiveSparkPlan (77) +AdaptiveSparkPlan (78) +- == Final Plan == - VeloxColumnarToRowExec (55) - +- ^ SortExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- VeloxAppendBatches (48) - +- ^ FilterExecTransformer (46) - +- ^ RegularHashAggregateExecTransformer (45) - +- ^ InputIteratorTransformer (44) - +- ShuffleQueryStage (42), Statistics(X) - +- ColumnarExchange (41) - +- ^ ProjectExecTransformer (39) - +- ^ FlushableHashAggregateExecTransformer (38) - +- ^ ProjectExecTransformer (37) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - :- ^ InputIteratorTransformer (26) - : +- ShuffleQueryStage (24), Statistics(X) - : +- ColumnarExchange (23) - : +- VeloxAppendBatches (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : :- ^ InputIteratorTransformer (9) - : : +- ShuffleQueryStage (7), Statistics(X) - : : +- ColumnarExchange (6) - : : +- VeloxAppendBatches (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (18) - : +- ShuffleQueryStage (16), Statistics(X) - : +- ColumnarExchange (15) - : +- VeloxAppendBatches (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FilterExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- VeloxAppendBatches (31) - +- ^ ProjectExecTransformer (29) - +- ^ FilterExecTransformer (28) - +- ^ Scan parquet (27) + VeloxColumnarToRowExec (56) + +- ^ SortExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ShuffleQueryStage (51), Statistics(X) + +- ColumnarExchange (50) + +- VeloxAppendBatches (49) + +- ^ FilterExecTransformer (47) + +- ^ RegularHashAggregateExecTransformer (46) + +- ^ InputIteratorTransformer (45) + +- ShuffleQueryStage (43), Statistics(X) + +- ColumnarExchange (42) + +- VeloxAppendBatches (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (76) - +- Exchange (75) - +- Filter (74) - +- HashAggregate (73) - +- Exchange (72) - +- HashAggregate (71) - +- Project (70) - +- ShuffledHashJoin Inner BuildRight (69) - :- Exchange (64) - : +- Project (63) - : +- ShuffledHashJoin Inner BuildRight (62) - : :- Exchange (58) - : : +- Filter (57) - : : +- Scan parquet (56) - : +- Exchange (61) - : +- Filter (60) - : +- Scan parquet (59) - +- Exchange (68) - +- Project (67) - +- Filter (66) - +- Scan parquet (65) + Sort (77) + +- Exchange (76) + +- Filter (75) + +- HashAggregate (74) + +- Exchange (73) + +- HashAggregate (72) + +- Project (71) + +- ShuffledHashJoin Inner BuildRight (70) + :- Exchange (65) + : +- Project (64) + : +- ShuffledHashJoin Inner BuildRight (63) + : :- Exchange (59) + : : +- Filter (58) + : : +- Scan parquet (57) + : +- Exchange (62) + : +- Filter (61) + : +- Scan parquet (60) + +- Exchange (69) + +- Project (68) + +- Filter (67) + +- Scan parquet (66) (1) Scan parquet @@ -233,432 +234,436 @@ Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(41) ColumnarExchange +(41) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(42) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(42) ShuffleQueryStage +(43) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(43) InputAdapter +(44) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(44) InputIteratorTransformer +(45) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(45) RegularHashAggregateExecTransformer +(46) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(46) FilterExecTransformer +(47) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(47) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(48) VeloxAppendBatches +(49) VeloxAppendBatches Input [2]: [ps_partkey#X, value#X] Arguments: X -(49) ColumnarExchange +(50) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(51) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(51) InputAdapter +(52) InputAdapter Input [2]: [ps_partkey#X, value#X] -(52) InputIteratorTransformer +(53) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(53) SortExecTransformer +(54) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(54) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(55) VeloxColumnarToRowExec +(56) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(56) Scan parquet +(57) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(57) Filter +(58) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(58) Exchange +(59) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) Scan parquet +(60) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(60) Filter +(61) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(61) Exchange +(62) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) ShuffledHashJoin +(63) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(63) Project +(64) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(64) Exchange +(65) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) Scan parquet +(66) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(67) Project +(68) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(68) Exchange +(69) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(69) ShuffledHashJoin +(70) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(70) Project +(71) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(71) HashAggregate +(72) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(72) Exchange +(73) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(73) HashAggregate +(74) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(74) Filter +(75) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(75) Exchange +(76) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) Sort +(77) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(77) AdaptiveSparkPlan +(78) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (127) +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (109) - +- ^ ProjectExecTransformer (107) - +- ^ RegularHashAggregateExecTransformer (106) - +- ^ RegularHashAggregateExecTransformer (105) - +- ^ ProjectExecTransformer (104) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (103) - :- ^ InputIteratorTransformer (98) - : +- ShuffleQueryStage (96), Statistics(X) - : +- ColumnarExchange (95) - : +- VeloxAppendBatches (94) - : +- ^ ProjectExecTransformer (92) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (91) - : :- ^ InputIteratorTransformer (86) - : : +- ShuffleQueryStage (84), Statistics(X) - : : +- ColumnarExchange (83) - : : +- VeloxAppendBatches (82) - : : +- ^ ProjectExecTransformer (80) - : : +- ^ FilterExecTransformer (79) - : : +- ^ Scan parquet (78) - : +- ^ InputIteratorTransformer (90) - : +- ShuffleQueryStage (88), Statistics(X) - : +- ReusedExchange (87) - +- ^ InputIteratorTransformer (102) - +- ShuffleQueryStage (100), Statistics(X) - +- ReusedExchange (99) + VeloxColumnarToRowExec (110) + +- ^ ProjectExecTransformer (108) + +- ^ RegularHashAggregateExecTransformer (107) + +- ^ RegularHashAggregateExecTransformer (106) + +- ^ ProjectExecTransformer (105) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + :- ^ InputIteratorTransformer (99) + : +- ShuffleQueryStage (97), Statistics(X) + : +- ColumnarExchange (96) + : +- VeloxAppendBatches (95) + : +- ^ ProjectExecTransformer (93) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) + : :- ^ InputIteratorTransformer (87) + : : +- ShuffleQueryStage (85), Statistics(X) + : : +- ColumnarExchange (84) + : : +- VeloxAppendBatches (83) + : : +- ^ ProjectExecTransformer (81) + : : +- ^ FilterExecTransformer (80) + : : +- ^ Scan parquet (79) + : +- ^ InputIteratorTransformer (91) + : +- ShuffleQueryStage (89), Statistics(X) + : +- ReusedExchange (88) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ReusedExchange (100) +- == Initial Plan == - HashAggregate (126) - +- HashAggregate (125) - +- Project (124) - +- ShuffledHashJoin Inner BuildRight (123) - :- Exchange (118) - : +- Project (117) - : +- ShuffledHashJoin Inner BuildRight (116) - : :- Exchange (112) - : : +- Filter (111) - : : +- Scan parquet (110) - : +- Exchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- Exchange (122) - +- Project (121) - +- Filter (120) - +- Scan parquet (119) - - -(78) Scan parquet + HashAggregate (127) + +- HashAggregate (126) + +- Project (125) + +- ShuffledHashJoin Inner BuildRight (124) + :- Exchange (119) + : +- Project (118) + : +- ShuffledHashJoin Inner BuildRight (117) + : :- Exchange (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Exchange (116) + : +- Filter (115) + : +- Scan parquet (114) + +- Exchange (123) + +- Project (122) + +- Filter (121) + +- Scan parquet (120) + + +(79) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(79) FilterExecTransformer +(80) FilterExecTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: isnotnull(ps_suppkey#X) -(80) ProjectExecTransformer +(81) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(81) WholeStageCodegenTransformer (X) +(82) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(82) VeloxAppendBatches +(83) VeloxAppendBatches Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(83) ColumnarExchange +(84) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(84) ShuffleQueryStage +(85) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(85) InputAdapter +(86) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(86) InputIteratorTransformer +(87) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(87) ReusedExchange [Reuses operator id: 15] +(88) ReusedExchange [Reuses operator id: 15] Output [2]: [s_suppkey#X, s_nationkey#X] -(88) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(89) InputAdapter +(90) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(90) InputIteratorTransformer +(91) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(91) ShuffledHashJoinExecTransformer +(92) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(92) ProjectExecTransformer +(93) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(93) WholeStageCodegenTransformer (X) +(94) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(94) VeloxAppendBatches +(95) VeloxAppendBatches Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(95) ColumnarExchange +(96) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(97) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(97) InputAdapter +(98) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(98) InputIteratorTransformer +(99) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(99) ReusedExchange [Reuses operator id: 32] +(100) ReusedExchange [Reuses operator id: 32] Output [1]: [n_nationkey#X] -(100) ShuffleQueryStage +(101) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(101) InputAdapter +(102) InputAdapter Input [1]: [n_nationkey#X] -(102) InputIteratorTransformer +(103) InputIteratorTransformer Input [1]: [n_nationkey#X] -(103) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(104) ProjectExecTransformer +(105) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(105) RegularHashAggregateExecTransformer +(106) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(106) RegularHashAggregateExecTransformer +(107) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(107) ProjectExecTransformer +(108) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(108) WholeStageCodegenTransformer (X) +(109) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(109) VeloxColumnarToRowExec +(110) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(110) Scan parquet +(111) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(112) Exchange +(113) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) Scan parquet +(114) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(114) Filter +(115) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(115) Exchange +(116) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(117) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(117) Project +(118) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(118) Exchange +(119) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) Scan parquet +(120) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(120) Filter +(121) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(121) Project +(122) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(122) Exchange +(123) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) ShuffledHashJoin +(124) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(124) Project +(125) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(125) HashAggregate +(126) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(126) HashAggregate +(127) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(127) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt index 0ee1a20a6c39..194b60bb7713 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt @@ -1,49 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32), Statistics(X) - +- ColumnarExchange (31) - +- VeloxAppendBatches (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7), Statistics(X) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- VeloxAppendBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ FilterExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- Project (46) - +- ShuffledHashJoin Inner BuildLeft (45) - :- Exchange (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Exchange (44) - +- Project (43) - +- Filter (42) - +- Scan parquet (41) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin Inner BuildLeft (46) + :- Exchange (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -144,129 +145,133 @@ Input [3]: [l_shipmode#X, sum#X, sum#X] Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(24) ColumnarExchange +(24) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(25) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(28) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(29) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(30) VeloxAppendBatches +(31) VeloxAppendBatches Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(31) ColumnarExchange +(32) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) SortExecTransformer +(36) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(37) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(38) Scan parquet +(39) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(40) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(42) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(42) Filter +(43) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(43) Project +(44) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(44) Exchange +(45) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(46) Project +(47) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(47) HashAggregate +(48) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(48) Exchange +(49) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(50) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(50) Exchange +(51) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(52) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(52) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt index be5442000bde..a9d4e199cfd2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (55) +AdaptiveSparkPlan (56) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- ^ SortExecTransformer (37) - +- ^ InputIteratorTransformer (36) - +- ShuffleQueryStage (34), Statistics(X) - +- ColumnarExchange (33) - +- VeloxAppendBatches (32) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- VeloxAppendBatches (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (54) - +- Exchange (53) - +- HashAggregate (52) - +- Exchange (51) - +- HashAggregate (50) - +- HashAggregate (49) - +- HashAggregate (48) - +- Project (47) - +- ShuffledHashJoin LeftOuter BuildRight (46) - :- Exchange (41) - : +- Scan parquet (40) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftOuter BuildRight (47) + :- Exchange (42) + : +- Scan parquet (41) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -160,138 +161,142 @@ Input [2]: [c_count#X, count#X] Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [2]: [c_count#X, count#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(32) VeloxAppendBatches +(33) VeloxAppendBatches Input [2]: [c_count#X, custdist#X] Arguments: X -(33) ColumnarExchange +(34) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(34) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(35) InputAdapter +(36) InputAdapter Input [2]: [c_count#X, custdist#X] -(36) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(37) SortExecTransformer +(38) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(38) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(39) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(40) Scan parquet +(41) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(41) Exchange +(42) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Scan parquet +(43) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(43) Filter +(44) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(44) Project +(45) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(45) Exchange +(46) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(47) Project +(48) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(48) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(49) HashAggregate +(50) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(50) HashAggregate +(51) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(51) Exchange +(52) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) HashAggregate +(53) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(53) Exchange +(54) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Sort +(55) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(55) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt index 7b33a6b69bd9..fec017400c11 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt @@ -1,13 +1,13 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- AQEShuffleRead (28) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- VeloxAppendBatches (25) - +- ^ ProjectExecTransformer (23) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (22) + VeloxColumnarToRowExec (30) + +- AQEShuffleRead (29) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (23) :- ^ InputIteratorTransformer (9) : +- ShuffleQueryStage (7), Statistics(X) : +- ColumnarExchange (6) @@ -15,31 +15,32 @@ AdaptiveSparkPlan (44) : +- ^ ProjectExecTransformer (3) : +- ^ FilterExecTransformer (2) : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ FilterExecTransformer (11) - +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- Project (41) - +- ShuffledHashJoin Inner BuildLeft (40) - :- Exchange (32) - : +- Filter (31) - : +- Scan parquet (30) - +- Filter (39) - +- HashAggregate (38) - +- Exchange (37) - +- HashAggregate (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (44) + +- Exchange (43) + +- Project (42) + +- ShuffledHashJoin Inner BuildLeft (41) + :- Exchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Filter (40) + +- HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -109,288 +110,297 @@ Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(16) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(21) FilterExecTransformer +(22) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(22) ShuffledHashJoinExecTransformer +(23) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(23) ProjectExecTransformer +(24) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(24) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(25) VeloxAppendBatches +(26) VeloxAppendBatches Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) ColumnarExchange +(27) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(28) AQEShuffleRead +(29) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(30) Scan parquet +(31) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(32) Exchange +(33) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(33) Scan parquet +(34) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(34) Filter +(35) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(35) Project +(36) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(36) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(37) Exchange +(38) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) HashAggregate +(39) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(39) Filter +(40) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(40) ShuffledHashJoin +(41) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(41) Project +(42) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(42) Exchange +(43) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(44) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 21 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (69) +Subquery:1 Hosting operator id = 22 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (71) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ RegularHashAggregateExecTransformer (58) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ ProjectExecTransformer (56) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ InputIteratorTransformer (54) - +- ShuffleQueryStage (52), Statistics(X) - +- ColumnarExchange (51) - +- ^ ProjectExecTransformer (49) - +- ^ FlushableHashAggregateExecTransformer (48) - +- ^ ProjectExecTransformer (47) - +- ^ FilterExecTransformer (46) - +- ^ Scan parquet (45) + VeloxColumnarToRowExec (62) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ ProjectExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ FilterExecTransformer (47) + +- ^ Scan parquet (46) +- == Initial Plan == - HashAggregate (68) - +- HashAggregate (67) - +- HashAggregate (66) - +- Exchange (65) - +- HashAggregate (64) - +- Project (63) - +- Filter (62) - +- Scan parquet (61) + HashAggregate (70) + +- HashAggregate (69) + +- HashAggregate (68) + +- Exchange (67) + +- HashAggregate (66) + +- Project (65) + +- Filter (64) + +- Scan parquet (63) -(45) Scan parquet +(46) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(46) FilterExecTransformer +(47) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(47) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(48) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(49) ProjectExecTransformer +(50) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(50) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(51) ColumnarExchange +(52) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(53) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(54) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(53) InputAdapter +(55) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(54) InputIteratorTransformer +(56) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(55) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(56) ProjectExecTransformer +(58) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(57) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(58) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(59) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(60) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(61) Scan parquet +(63) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(62) Filter +(64) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(63) Project +(65) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(64) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(65) Exchange +(67) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) HashAggregate +(68) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(67) HashAggregate +(69) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(68) HashAggregate +(70) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(69) AdaptiveSparkPlan +(71) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt index 287ae9f00ed1..15dd2fa6da8e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt @@ -1,62 +1,64 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (45) - +- ^ SortExecTransformer (43) - +- ^ InputIteratorTransformer (42) - +- ShuffleQueryStage (40), Statistics(X) - +- ColumnarExchange (39) - +- VeloxAppendBatches (38) - +- ^ RegularHashAggregateExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ ProjectExecTransformer (30) - +- ^ FlushableHashAggregateExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7), Statistics(X) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- VeloxAppendBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ FilterExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (47) + +- ^ SortExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42), Statistics(X) + +- ColumnarExchange (41) + +- VeloxAppendBatches (40) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ ProjectExecTransformer (31) + +- ^ FlushableHashAggregateExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (66) - +- Exchange (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- Project (58) - +- ShuffledHashJoin Inner BuildRight (57) - :- Exchange (53) - : +- BroadcastHashJoin LeftAnti BuildRight (52) - : :- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (51) - : +- Project (50) - : +- Filter (49) - : +- Scan parquet (48) - +- Exchange (56) - +- Filter (55) - +- Scan parquet (54) + Sort (68) + +- Exchange (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- HashAggregate (63) + +- Exchange (62) + +- HashAggregate (61) + +- Project (60) + +- ShuffledHashJoin Inner BuildRight (59) + :- Exchange (55) + : +- BroadcastHashJoin LeftAnti BuildRight (54) + : :- Filter (49) + : : +- Scan parquet (48) + : +- BroadcastExchange (53) + : +- Project (52) + : +- Filter (51) + : +- Scan parquet (50) + +- Exchange (58) + +- Filter (57) + +- Scan parquet (56) (1) Scan parquet @@ -157,203 +159,211 @@ Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(24) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(28) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(29) FlushableHashAggregateExecTransformer +(30) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(30) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(32) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(34) InputAdapter +(36) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(35) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(36) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(37) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(38) VeloxAppendBatches +(40) VeloxAppendBatches Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(39) ColumnarExchange +(41) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(42) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(41) InputAdapter +(43) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(42) InputIteratorTransformer +(44) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) SortExecTransformer +(45) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(44) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(45) VeloxColumnarToRowExec +(47) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(46) Scan parquet +(48) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(47) Filter +(49) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(48) Scan parquet +(50) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(49) Filter +(51) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(50) Project +(52) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(51) BroadcastExchange +(53) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(52) BroadcastHashJoin +(54) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(53) Exchange +(55) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(56) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(56) Exchange +(58) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(59) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(58) Project +(60) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(59) HashAggregate +(61) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(60) Exchange +(62) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(63) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(62) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(63) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(66) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(65) Exchange +(67) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) Sort +(68) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(67) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt index 23d34ad9b587..69af0fd38e92 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt @@ -1,12 +1,12 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- ^ ProjectExecTransformer (37) - +- ^ RegularHashAggregateExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ ProjectExecTransformer (34) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (33) + VeloxColumnarToRowExec (40) + +- ^ ProjectExecTransformer (38) + +- ^ RegularHashAggregateExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ ProjectExecTransformer (35) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (34) :- ^ ProjectExecTransformer (20) : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) : :- ^ InputIteratorTransformer (9) @@ -23,36 +23,37 @@ AdaptiveSparkPlan (59) : +- ^ ProjectExecTransformer (12) : +- ^ FilterExecTransformer (11) : +- ^ Scan parquet (10) - +- ^ FilterExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ FilterExecTransformer (22) - +- ^ Scan parquet (21) + +- ^ FilterExecTransformer (33) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ FilterExecTransformer (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (58) - +- HashAggregate (57) - +- Project (56) - +- ShuffledHashJoin Inner BuildRight (55) - :- Project (48) - : +- ShuffledHashJoin Inner BuildRight (47) - : :- Exchange (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Project (45) - : +- Filter (44) - : +- Scan parquet (43) - +- Filter (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- Filter (50) - +- Scan parquet (49) + HashAggregate (59) + +- HashAggregate (58) + +- Project (57) + +- ShuffledHashJoin Inner BuildRight (56) + :- Project (49) + : +- ShuffledHashJoin Inner BuildRight (48) + : :- Exchange (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Exchange (47) + : +- Project (46) + : +- Filter (45) + : +- Scan parquet (44) + +- Filter (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- Filter (51) + +- Scan parquet (50) (1) Scan parquet @@ -164,168 +165,172 @@ Input [3]: [l_partkey#X, sum#X, count#X] Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(32) FilterExecTransformer +(33) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(33) ShuffledHashJoinExecTransformer +(34) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(34) ProjectExecTransformer +(35) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(35) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(36) RegularHashAggregateExecTransformer +(37) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(37) ProjectExecTransformer +(38) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(38) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(39) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(40) Scan parquet +(41) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(41) Filter +(42) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(42) Exchange +(43) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Scan parquet +(44) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(45) Project +(46) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(46) Exchange +(47) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(48) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(48) Project +(49) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(49) Scan parquet +(50) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(50) Filter +(51) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(51) HashAggregate +(52) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(52) Exchange +(53) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(54) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(54) Filter +(55) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(55) ShuffledHashJoin +(56) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(56) Project +(57) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(57) HashAggregate +(58) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(58) HashAggregate +(59) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] -(59) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt index fdf15018cafa..8db05ed7572c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt @@ -1,18 +1,18 @@ == Physical Plan == -AdaptiveSparkPlan (102) +AdaptiveSparkPlan (103) +- == Final Plan == - VeloxColumnarToRowExec (69) - +- TakeOrderedAndProjectExecTransformer (68) - +- ^ RegularHashAggregateExecTransformer (66) - +- ^ RegularHashAggregateExecTransformer (65) - +- ^ ProjectExecTransformer (64) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (63) - :- ^ InputIteratorTransformer (45) - : +- ShuffleQueryStage (43), Statistics(X) - : +- ColumnarExchange (42) - : +- VeloxAppendBatches (41) - : +- ^ ProjectExecTransformer (39) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (38) + VeloxColumnarToRowExec (70) + +- TakeOrderedAndProjectExecTransformer (69) + +- ^ RegularHashAggregateExecTransformer (67) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ ProjectExecTransformer (65) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (64) + :- ^ InputIteratorTransformer (46) + : +- ShuffleQueryStage (44), Statistics(X) + : +- ColumnarExchange (43) + : +- VeloxAppendBatches (42) + : +- ^ ProjectExecTransformer (40) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (39) : :- ^ InputIteratorTransformer (9) : : +- ShuffleQueryStage (7), Statistics(X) : : +- ColumnarExchange (6) @@ -20,12 +20,12 @@ AdaptiveSparkPlan (102) : : +- ^ ProjectExecTransformer (3) : : +- ^ FilterExecTransformer (2) : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (37) - : +- ShuffleQueryStage (35), Statistics(X) - : +- ColumnarExchange (34) - : +- VeloxAppendBatches (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (30) + : +- ^ InputIteratorTransformer (38) + : +- ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- VeloxAppendBatches (34) + : +- ^ ProjectExecTransformer (32) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (31) : :- ^ InputIteratorTransformer (18) : : +- ShuffleQueryStage (16), Statistics(X) : : +- ColumnarExchange (15) @@ -33,62 +33,63 @@ AdaptiveSparkPlan (102) : : +- ^ ProjectExecTransformer (12) : : +- ^ FilterExecTransformer (11) : : +- ^ Scan parquet (10) - : +- ^ ProjectExecTransformer (29) - : +- ^ FilterExecTransformer (28) - : +- ^ RegularHashAggregateExecTransformer (27) - : +- ^ InputIteratorTransformer (26) - : +- ShuffleQueryStage (24), Statistics(X) - : +- ColumnarExchange (23) - : +- ^ ProjectExecTransformer (21) - : +- ^ FlushableHashAggregateExecTransformer (20) - : +- ^ Scan parquet (19) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - :- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- VeloxAppendBatches (50) - : +- ^ ProjectExecTransformer (48) - : +- ^ FilterExecTransformer (47) - : +- ^ Scan parquet (46) - +- ^ ProjectExecTransformer (61) - +- ^ FilterExecTransformer (60) - +- ^ RegularHashAggregateExecTransformer (59) - +- ^ InputIteratorTransformer (58) - +- ShuffleQueryStage (56), Statistics(X) - +- ReusedExchange (55) + : +- ^ ProjectExecTransformer (30) + : +- ^ FilterExecTransformer (29) + : +- ^ RegularHashAggregateExecTransformer (28) + : +- ^ InputIteratorTransformer (27) + : +- ShuffleQueryStage (25), Statistics(X) + : +- ColumnarExchange (24) + : +- VeloxAppendBatches (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (63) + :- ^ InputIteratorTransformer (55) + : +- ShuffleQueryStage (53), Statistics(X) + : +- ColumnarExchange (52) + : +- VeloxAppendBatches (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ Scan parquet (47) + +- ^ ProjectExecTransformer (62) + +- ^ FilterExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57), Statistics(X) + +- ReusedExchange (56) +- == Initial Plan == - TakeOrderedAndProject (101) - +- HashAggregate (100) - +- HashAggregate (99) - +- Project (98) - +- ShuffledHashJoin Inner BuildRight (97) - :- Exchange (86) - : +- Project (85) - : +- ShuffledHashJoin Inner BuildLeft (84) - : :- Exchange (72) - : : +- Filter (71) - : : +- Scan parquet (70) - : +- Exchange (83) - : +- ShuffledHashJoin LeftSemi BuildRight (82) - : :- Exchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- Project (81) - : +- Filter (80) - : +- HashAggregate (79) - : +- Exchange (78) - : +- HashAggregate (77) - : +- Scan parquet (76) - +- ShuffledHashJoin LeftSemi BuildRight (96) - :- Exchange (89) - : +- Filter (88) - : +- Scan parquet (87) - +- Project (95) - +- Filter (94) - +- HashAggregate (93) - +- Exchange (92) - +- HashAggregate (91) - +- Scan parquet (90) + TakeOrderedAndProject (102) + +- HashAggregate (101) + +- HashAggregate (100) + +- Project (99) + +- ShuffledHashJoin Inner BuildRight (98) + :- Exchange (87) + : +- Project (86) + : +- ShuffledHashJoin Inner BuildLeft (85) + : :- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (84) + : +- ShuffledHashJoin LeftSemi BuildRight (83) + : :- Exchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Project (82) + : +- Filter (81) + : +- HashAggregate (80) + : +- Exchange (79) + : +- HashAggregate (78) + : +- Scan parquet (77) + +- ShuffledHashJoin LeftSemi BuildRight (97) + :- Exchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- Project (96) + +- Filter (95) + +- HashAggregate (94) + +- Exchange (93) + +- HashAggregate (92) + +- Scan parquet (91) (1) Scan parquet @@ -186,364 +187,368 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(28) FilterExecTransformer +(29) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(29) ProjectExecTransformer +(30) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(30) ShuffledHashJoinExecTransformer +(31) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(33) VeloxAppendBatches +(34) VeloxAppendBatches Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(34) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(35) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(36) InputAdapter +(37) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(38) ShuffledHashJoinExecTransformer +(39) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(39) ProjectExecTransformer +(40) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(40) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(41) VeloxAppendBatches +(42) VeloxAppendBatches Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(42) ColumnarExchange +(43) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(43) ShuffleQueryStage +(44) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(44) InputAdapter +(45) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(45) InputIteratorTransformer +(46) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(46) Scan parquet +(47) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(47) FilterExecTransformer +(48) FilterExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Arguments: isnotnull(l_orderkey#X) -(48) ProjectExecTransformer +(49) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(49) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(50) VeloxAppendBatches +(51) VeloxAppendBatches Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: X -(51) ColumnarExchange +(52) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(53) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(53) InputAdapter +(54) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(54) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(55) ReusedExchange [Reuses operator id: 23] +(56) ReusedExchange [Reuses operator id: 24] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(56) ShuffleQueryStage +(57) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(57) InputAdapter +(58) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(58) InputIteratorTransformer +(59) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(59) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(60) FilterExecTransformer +(61) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(61) ProjectExecTransformer +(62) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(62) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(63) ShuffledHashJoinExecTransformer +(64) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(64) ProjectExecTransformer +(65) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(65) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(66) RegularHashAggregateExecTransformer +(67) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(67) WholeStageCodegenTransformer (X) +(68) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(68) TakeOrderedAndProjectExecTransformer +(69) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(69) VeloxColumnarToRowExec +(70) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(70) Scan parquet +(71) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(72) Exchange +(73) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(73) Scan parquet +(74) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(75) Exchange +(76) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) Scan parquet +(77) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(77) HashAggregate +(78) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(78) Exchange +(79) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) HashAggregate +(80) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(80) Filter +(81) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(81) Project +(82) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(82) ShuffledHashJoin +(83) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(83) Exchange +(84) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) ShuffledHashJoin +(85) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(85) Project +(86) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(86) Exchange +(87) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Scan parquet +(88) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(88) Filter +(89) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(89) Exchange +(90) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(91) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(91) HashAggregate +(92) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(92) Exchange +(93) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) HashAggregate +(94) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(94) Filter +(95) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(95) Project +(96) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(96) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(97) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(98) Project +(99) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(99) HashAggregate +(100) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(100) HashAggregate +(101) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(101) TakeOrderedAndProject +(102) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(102) AdaptiveSparkPlan +(103) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt index 245faea0cde8..5c05ec24757e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt @@ -1,112 +1,113 @@ == Physical Plan == -AdaptiveSparkPlan (127) +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ RegularHashAggregateExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ ProjectExecTransformer (82) - +- ^ FlushableHashAggregateExecTransformer (81) - +- ^ ProjectExecTransformer (80) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) - :- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67), Statistics(X) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50), Statistics(X) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) - : : :- ^ InputIteratorTransformer (9) - : : : +- ShuffleQueryStage (7), Statistics(X) - : : : +- ColumnarExchange (6) - : : : +- VeloxAppendBatches (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (44) - : : +- ShuffleQueryStage (42), Statistics(X) - : : +- ColumnarExchange (41) - : : +- VeloxAppendBatches (40) - : : +- ^ ProjectExecTransformer (38) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) - : : : :- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ FilterExecTransformer (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (26) - : : : +- ShuffleQueryStage (24), Statistics(X) - : : : +- ColumnarExchange (23) - : : : +- VeloxAppendBatches (22) - : : : +- ^ ProjectExecTransformer (20) - : : : +- ^ Scan parquet (19) - : : +- ^ InputIteratorTransformer (36) - : : +- ShuffleQueryStage (34), Statistics(X) - : : +- ColumnarExchange (33) - : : +- VeloxAppendBatches (32) - : : +- ^ ProjectExecTransformer (30) - : : +- ^ FilterExecTransformer (29) - : : +- ^ Scan parquet (28) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- VeloxAppendBatches (57) - : +- ^ ProjectExecTransformer (55) - : +- ^ FilterExecTransformer (54) - : +- ^ Scan parquet (53) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- VeloxAppendBatches (74) - +- ^ ProjectExecTransformer (72) - +- ^ FilterExecTransformer (71) - +- ^ Scan parquet (70) + VeloxColumnarToRowExec (91) + +- ^ RegularHashAggregateExecTransformer (89) + +- ^ InputIteratorTransformer (88) + +- ShuffleQueryStage (86), Statistics(X) + +- ColumnarExchange (85) + +- VeloxAppendBatches (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24), Statistics(X) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34), Statistics(X) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ FilterExecTransformer (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ FilterExecTransformer (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ FilterExecTransformer (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (116) - : +- Project (115) - : +- ShuffledHashJoin Inner BuildRight (114) - : :- Exchange (109) - : : +- Project (108) - : : +- ShuffledHashJoin Inner BuildLeft (107) - : : :- Exchange (93) - : : : +- Filter (92) - : : : +- Scan parquet (91) - : : +- Exchange (106) - : : +- ShuffledHashJoin LeftAnti BuildRight (105) - : : :- ShuffledHashJoin LeftSemi BuildRight (100) - : : : :- Exchange (97) - : : : : +- Project (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (99) - : : : +- Scan parquet (98) - : : +- Exchange (104) - : : +- Project (103) - : : +- Filter (102) - : : +- Scan parquet (101) - : +- Exchange (113) - : +- Project (112) - : +- Filter (111) - : +- Scan parquet (110) - +- Exchange (120) - +- Project (119) - +- Filter (118) - +- Scan parquet (117) + TakeOrderedAndProject (127) + +- HashAggregate (126) + +- Exchange (125) + +- HashAggregate (124) + +- Project (123) + +- ShuffledHashJoin Inner BuildRight (122) + :- Exchange (117) + : +- Project (116) + : +- ShuffledHashJoin Inner BuildRight (115) + : :- Exchange (110) + : : +- Project (109) + : : +- ShuffledHashJoin Inner BuildLeft (108) + : : :- Exchange (94) + : : : +- Filter (93) + : : : +- Scan parquet (92) + : : +- Exchange (107) + : : +- ShuffledHashJoin LeftAnti BuildRight (106) + : : :- ShuffledHashJoin LeftSemi BuildRight (101) + : : : :- Exchange (98) + : : : : +- Project (97) + : : : : +- Filter (96) + : : : : +- Scan parquet (95) + : : : +- Exchange (100) + : : : +- Scan parquet (99) + : : +- Exchange (105) + : : +- Project (104) + : : +- Filter (103) + : : +- Scan parquet (102) + : +- Exchange (114) + : +- Project (113) + : +- Filter (112) + : +- Scan parquet (111) + +- Exchange (121) + +- Project (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -448,206 +449,210 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(84) ColumnarExchange +(84) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(85) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(86) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(86) InputAdapter +(87) InputAdapter Input [2]: [s_name#X, count#X] -(87) InputIteratorTransformer +(88) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(88) RegularHashAggregateExecTransformer +(89) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(89) WholeStageCodegenTransformer (X) +(90) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(90) VeloxColumnarToRowExec +(91) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(91) Scan parquet +(92) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(93) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(94) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(95) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(96) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Project +(97) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(97) Exchange +(98) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(99) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(99) Exchange +(100) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) ShuffledHashJoin +(101) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(101) Scan parquet +(102) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(102) Filter +(103) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(103) Project +(104) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(104) Exchange +(105) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) ShuffledHashJoin +(106) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(106) Exchange +(107) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(108) Project +(109) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(109) Exchange +(110) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(112) Project +(113) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(113) Exchange +(114) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(115) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(115) Project +(116) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(116) Exchange +(117) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(118) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(118) Filter +(119) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(119) Project +(120) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(120) Exchange +(121) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(122) Project +(123) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(123) HashAggregate +(124) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(124) Exchange +(125) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(126) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(126) TakeOrderedAndProject +(127) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(127) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt index 2a822968008a..af5c086c274a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt @@ -1,46 +1,47 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (50) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- VeloxAppendBatches (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7), Statistics(X) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ ProjectExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin LeftAnti BuildRight (42) - :- Exchange (39) - : +- Filter (38) - : +- Scan parquet (37) - +- Exchange (41) - +- Scan parquet (40) + Sort (49) + +- Exchange (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftAnti BuildRight (43) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (42) + +- Scan parquet (41) (1) Scan parquet @@ -136,254 +137,264 @@ Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(29) VeloxAppendBatches +(30) VeloxAppendBatches Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(30) ColumnarExchange +(31) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(32) InputAdapter +(33) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(33) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) SortExecTransformer +(35) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(35) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(36) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(37) Scan parquet +(38) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(38) Filter +(39) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(39) Exchange +(40) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) Scan parquet +(41) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(41) Exchange +(42) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(44) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(45) Exchange +(46) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(47) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(47) Exchange +(48) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(49) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(50) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (68) +AdaptiveSparkPlan (70) +- == Final Plan == - VeloxColumnarToRowExec (61) - +- ^ RegularHashAggregateExecTransformer (59) - +- ^ InputIteratorTransformer (58) - +- ShuffleQueryStage (56), Statistics(X) - +- ColumnarExchange (55) - +- ^ FlushableHashAggregateExecTransformer (53) - +- ^ ProjectExecTransformer (52) - +- ^ FilterExecTransformer (51) - +- ^ Scan parquet (50) + VeloxColumnarToRowExec (63) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ InputIteratorTransformer (60) + +- ShuffleQueryStage (58), Statistics(X) + +- ColumnarExchange (57) + +- VeloxAppendBatches (56) + +- ^ FlushableHashAggregateExecTransformer (54) + +- ^ ProjectExecTransformer (53) + +- ^ FilterExecTransformer (52) + +- ^ Scan parquet (51) +- == Initial Plan == - HashAggregate (67) - +- Exchange (66) - +- HashAggregate (65) - +- Project (64) - +- Filter (63) - +- Scan parquet (62) + HashAggregate (69) + +- Exchange (68) + +- HashAggregate (67) + +- Project (66) + +- Filter (65) + +- Scan parquet (64) -(50) Scan parquet +(51) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(51) FilterExecTransformer +(52) FilterExecTransformer Input [2]: [c_phone#X, c_acctbal#X] Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(52) ProjectExecTransformer +(53) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(53) FlushableHashAggregateExecTransformer +(54) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(54) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(55) ColumnarExchange +(56) VeloxAppendBatches +Input [2]: [sum#X, count#X] +Arguments: X + +(57) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(56) ShuffleQueryStage +(58) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(57) InputAdapter +(59) InputAdapter Input [2]: [sum#X, count#X] -(58) InputIteratorTransformer +(60) InputIteratorTransformer Input [2]: [sum#X, count#X] -(59) RegularHashAggregateExecTransformer +(61) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(60) WholeStageCodegenTransformer (X) +(62) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(61) VeloxColumnarToRowExec +(63) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(62) Scan parquet +(64) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(63) Filter +(65) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(64) Project +(66) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(65) HashAggregate +(67) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(66) Exchange +(68) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(67) HashAggregate +(69) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(68) AdaptiveSparkPlan +(70) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (68) +AdaptiveSparkPlan (70) +- == Final Plan == - VeloxColumnarToRowExec (61) - +- ^ RegularHashAggregateExecTransformer (59) - +- ^ InputIteratorTransformer (58) - +- ShuffleQueryStage (56), Statistics(X) - +- ColumnarExchange (55) - +- ^ FlushableHashAggregateExecTransformer (53) - +- ^ ProjectExecTransformer (52) - +- ^ FilterExecTransformer (51) - +- ^ Scan parquet (50) + VeloxColumnarToRowExec (63) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ InputIteratorTransformer (60) + +- ShuffleQueryStage (58), Statistics(X) + +- ColumnarExchange (57) + +- VeloxAppendBatches (56) + +- ^ FlushableHashAggregateExecTransformer (54) + +- ^ ProjectExecTransformer (53) + +- ^ FilterExecTransformer (52) + +- ^ Scan parquet (51) +- == Initial Plan == - HashAggregate (67) - +- Exchange (66) - +- HashAggregate (65) - +- Project (64) - +- Filter (63) - +- Scan parquet (62) \ No newline at end of file + HashAggregate (69) + +- Exchange (68) + +- HashAggregate (67) + +- Project (66) + +- Filter (65) + +- Scan parquet (64) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt index deba9b23f1fa..4da32d7f70ac 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt @@ -1,50 +1,51 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32), Statistics(X) - +- ColumnarExchange (31) - +- VeloxAppendBatches (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7), Statistics(X) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- VeloxAppendBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ FilterExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (52) - +- Exchange (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- ShuffledHashJoin LeftSemi BuildRight (46) - :- Exchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (53) + +- Exchange (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftSemi BuildRight (47) + :- Exchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -145,133 +146,137 @@ Input [2]: [o_orderpriority#X, count#X] Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(24) ColumnarExchange +(24) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(28) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(29) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(30) VeloxAppendBatches +(31) VeloxAppendBatches Input [2]: [o_orderpriority#X, order_count#X] Arguments: X -(31) ColumnarExchange +(32) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(35) SortExecTransformer +(36) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(37) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(41) Exchange +(42) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Scan parquet +(43) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(43) Filter +(44) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(44) Project +(45) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(45) Exchange +(46) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(47) Project +(48) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(48) HashAggregate +(49) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(49) Exchange +(50) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(51) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(51) Exchange +(52) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(53) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt index 257c7a998c7e..2669a9fce3ae 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt @@ -1,126 +1,127 @@ == Physical Plan == -AdaptiveSparkPlan (145) +AdaptiveSparkPlan (146) +- == Final Plan == - VeloxColumnarToRowExec (105) - +- ^ SortExecTransformer (103) - +- ^ InputIteratorTransformer (102) - +- ShuffleQueryStage (100), Statistics(X) - +- ColumnarExchange (99) - +- VeloxAppendBatches (98) - +- ^ RegularHashAggregateExecTransformer (96) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ColumnarExchange (92) - +- ^ ProjectExecTransformer (90) - +- ^ FlushableHashAggregateExecTransformer (89) - +- ^ ProjectExecTransformer (88) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - :- ^ InputIteratorTransformer (77) - : +- ShuffleQueryStage (75), Statistics(X) - : +- ColumnarExchange (74) - : +- VeloxAppendBatches (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - : :- ^ InputIteratorTransformer (60) - : : +- ShuffleQueryStage (58), Statistics(X) - : : +- ColumnarExchange (57) - : : +- VeloxAppendBatches (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - : : :- ^ InputIteratorTransformer (43) - : : : +- ShuffleQueryStage (41), Statistics(X) - : : : +- ColumnarExchange (40) - : : : +- VeloxAppendBatches (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - : : : :- ^ InputIteratorTransformer (26) - : : : : +- ShuffleQueryStage (24), Statistics(X) - : : : : +- ColumnarExchange (23) - : : : : +- VeloxAppendBatches (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ InputIteratorTransformer (9) - : : : : : +- ShuffleQueryStage (7), Statistics(X) - : : : : : +- ColumnarExchange (6) - : : : : : +- VeloxAppendBatches (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ FilterExecTransformer (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (35) - : : : +- ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- VeloxAppendBatches (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ FilterExecTransformer (28) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50), Statistics(X) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ FilterExecTransformer (45) - : : +- ^ Scan parquet (44) - : +- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67), Statistics(X) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ FilterExecTransformer (62) - : +- ^ Scan parquet (61) - +- ^ InputIteratorTransformer (86) - +- ShuffleQueryStage (84), Statistics(X) - +- ColumnarExchange (83) - +- VeloxAppendBatches (82) - +- ^ ProjectExecTransformer (80) - +- ^ FilterExecTransformer (79) - +- ^ Scan parquet (78) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94), Statistics(X) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (144) - +- Exchange (143) - +- HashAggregate (142) - +- Exchange (141) - +- HashAggregate (140) - +- Project (139) - +- ShuffledHashJoin Inner BuildRight (138) - :- Exchange (133) - : +- Project (132) - : +- ShuffledHashJoin Inner BuildRight (131) - : :- Exchange (127) - : : +- Project (126) - : : +- ShuffledHashJoin Inner BuildRight (125) - : : :- Exchange (121) - : : : +- Project (120) - : : : +- ShuffledHashJoin Inner BuildRight (119) - : : : :- Exchange (115) - : : : : +- Project (114) - : : : : +- ShuffledHashJoin Inner BuildLeft (113) - : : : : :- Exchange (108) - : : : : : +- Filter (107) - : : : : : +- Scan parquet (106) - : : : : +- Exchange (112) - : : : : +- Project (111) - : : : : +- Filter (110) - : : : : +- Scan parquet (109) - : : : +- Exchange (118) - : : : +- Filter (117) - : : : +- Scan parquet (116) - : : +- Exchange (124) - : : +- Filter (123) - : : +- Scan parquet (122) - : +- Exchange (130) - : +- Filter (129) - : +- Scan parquet (128) - +- Exchange (137) - +- Project (136) - +- Filter (135) - +- Scan parquet (134) + Sort (145) + +- Exchange (144) + +- HashAggregate (143) + +- Exchange (142) + +- HashAggregate (141) + +- Project (140) + +- ShuffledHashJoin Inner BuildRight (139) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Project (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (138) + +- Project (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -493,245 +494,249 @@ Input [3]: [n_name#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(92) ColumnarExchange +(92) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(93) ShuffleQueryStage +(94) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(94) InputAdapter +(95) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(95) InputIteratorTransformer +(96) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(96) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(97) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(98) VeloxAppendBatches +(99) VeloxAppendBatches Input [2]: [n_name#X, revenue#X] Arguments: X -(99) ColumnarExchange +(100) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(100) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(101) InputAdapter +(102) InputAdapter Input [2]: [n_name#X, revenue#X] -(102) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(103) SortExecTransformer +(104) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(104) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(105) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(106) Scan parquet +(107) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(108) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(109) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Scan parquet +(110) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(110) Filter +(111) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(111) Project +(112) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(112) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(114) Project +(115) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(115) Exchange +(116) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) Scan parquet +(117) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(117) Filter +(118) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(118) Exchange +(119) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(120) Project +(121) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(121) Exchange +(122) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(123) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(123) Filter +(124) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(124) Exchange +(125) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(126) Project +(127) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(127) Exchange +(128) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(129) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(129) Filter +(130) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(130) Exchange +(131) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(132) Project +(133) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(133) Exchange +(134) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(134) Scan parquet +(135) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(135) Filter +(136) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(136) Project +(137) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(137) Exchange +(138) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(138) ShuffledHashJoin +(139) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(139) Project +(140) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(140) HashAggregate +(141) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(141) Exchange +(142) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) HashAggregate +(143) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(143) Exchange +(144) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(144) Sort +(145) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(145) AdaptiveSparkPlan +(146) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/6.txt index 3a3148c1c113..3432579a0de0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/6.txt @@ -7,7 +7,7 @@ AdaptiveSparkPlan (20) +- ShuffleQueryStage (8), Statistics(X) +- ColumnarExchange (7) +- VeloxAppendBatches (6) - +- ^ RegularHashAggregateExecTransformer (4) + +- ^ FlushableHashAggregateExecTransformer (4) +- ^ ProjectExecTransformer (3) +- ^ FilterExecTransformer (2) +- ^ Scan parquet (1) @@ -35,7 +35,7 @@ Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnot Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) RegularHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt index 1255a3a0c6ec..b5abf7e36164 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt @@ -1,120 +1,121 @@ == Physical Plan == -AdaptiveSparkPlan (138) +AdaptiveSparkPlan (139) +- == Final Plan == - VeloxColumnarToRowExec (100) - +- ^ SortExecTransformer (98) - +- ^ InputIteratorTransformer (97) - +- ShuffleQueryStage (95), Statistics(X) - +- ColumnarExchange (94) - +- VeloxAppendBatches (93) - +- ^ RegularHashAggregateExecTransformer (91) - +- ^ InputIteratorTransformer (90) - +- ShuffleQueryStage (88), Statistics(X) - +- ColumnarExchange (87) - +- ^ ProjectExecTransformer (85) - +- ^ FlushableHashAggregateExecTransformer (84) - +- ^ ProjectExecTransformer (83) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) - :- ^ InputIteratorTransformer (77) - : +- ShuffleQueryStage (75), Statistics(X) - : +- ColumnarExchange (74) - : +- VeloxAppendBatches (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - : :- ^ InputIteratorTransformer (60) - : : +- ShuffleQueryStage (58), Statistics(X) - : : +- ColumnarExchange (57) - : : +- VeloxAppendBatches (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - : : :- ^ InputIteratorTransformer (43) - : : : +- ShuffleQueryStage (41), Statistics(X) - : : : +- ColumnarExchange (40) - : : : +- VeloxAppendBatches (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : :- ^ InputIteratorTransformer (26) - : : : : +- ShuffleQueryStage (24), Statistics(X) - : : : : +- ColumnarExchange (23) - : : : : +- VeloxAppendBatches (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ InputIteratorTransformer (9) - : : : : : +- ShuffleQueryStage (7), Statistics(X) - : : : : : +- ColumnarExchange (6) - : : : : : +- VeloxAppendBatches (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ FilterExecTransformer (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (35) - : : : +- ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- VeloxAppendBatches (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ FilterExecTransformer (28) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50), Statistics(X) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ FilterExecTransformer (45) - : : +- ^ Scan parquet (44) - : +- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67), Statistics(X) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ FilterExecTransformer (62) - : +- ^ Scan parquet (61) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ReusedExchange (78) + VeloxColumnarToRowExec (101) + +- ^ SortExecTransformer (99) + +- ^ InputIteratorTransformer (98) + +- ShuffleQueryStage (96), Statistics(X) + +- ColumnarExchange (95) + +- VeloxAppendBatches (94) + +- ^ RegularHashAggregateExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- VeloxAppendBatches (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ReusedExchange (78) +- == Initial Plan == - Sort (137) - +- Exchange (136) - +- HashAggregate (135) - +- Exchange (134) - +- HashAggregate (133) - +- Project (132) - +- ShuffledHashJoin Inner BuildRight (131) - :- Exchange (127) - : +- Project (126) - : +- ShuffledHashJoin Inner BuildRight (125) - : :- Exchange (121) - : : +- Project (120) - : : +- ShuffledHashJoin Inner BuildRight (119) - : : :- Exchange (115) - : : : +- Project (114) - : : : +- ShuffledHashJoin Inner BuildRight (113) - : : : :- Exchange (109) - : : : : +- Project (108) - : : : : +- ShuffledHashJoin Inner BuildLeft (107) - : : : : :- Exchange (103) - : : : : : +- Filter (102) - : : : : : +- Scan parquet (101) - : : : : +- Exchange (106) - : : : : +- Filter (105) - : : : : +- Scan parquet (104) - : : : +- Exchange (112) - : : : +- Filter (111) - : : : +- Scan parquet (110) - : : +- Exchange (118) - : : +- Filter (117) - : : +- Scan parquet (116) - : +- Exchange (124) - : +- Filter (123) - : +- Scan parquet (122) - +- Exchange (130) - +- Filter (129) - +- Scan parquet (128) + Sort (138) + +- Exchange (137) + +- HashAggregate (136) + +- Exchange (135) + +- HashAggregate (134) + +- Project (133) + +- ShuffledHashJoin Inner BuildRight (132) + :- Exchange (128) + : +- Project (127) + : +- ShuffledHashJoin Inner BuildRight (126) + : :- Exchange (122) + : : +- Project (121) + : : +- ShuffledHashJoin Inner BuildRight (120) + : : :- Exchange (116) + : : : +- Project (115) + : : : +- ShuffledHashJoin Inner BuildRight (114) + : : : :- Exchange (110) + : : : : +- Project (109) + : : : : +- ShuffledHashJoin Inner BuildLeft (108) + : : : : :- Exchange (104) + : : : : : +- Filter (103) + : : : : : +- Scan parquet (102) + : : : : +- Exchange (107) + : : : : +- Filter (106) + : : : : +- Scan parquet (105) + : : : +- Exchange (113) + : : : +- Filter (112) + : : : +- Scan parquet (111) + : : +- Exchange (119) + : : +- Filter (118) + : : +- Scan parquet (117) + : +- Exchange (125) + : +- Filter (124) + : +- Scan parquet (123) + +- Exchange (131) + +- Filter (130) + +- Scan parquet (129) (1) Scan parquet @@ -463,237 +464,241 @@ Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(87) ColumnarExchange +(87) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(88) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(88) ShuffleQueryStage +(89) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(89) InputAdapter +(90) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(90) InputIteratorTransformer +(91) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(91) RegularHashAggregateExecTransformer +(92) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(92) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(93) VeloxAppendBatches +(94) VeloxAppendBatches Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(94) ColumnarExchange +(95) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(95) ShuffleQueryStage +(96) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(96) InputAdapter +(97) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(97) InputIteratorTransformer +(98) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(98) SortExecTransformer +(99) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(99) WholeStageCodegenTransformer (X) +(100) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(100) VeloxColumnarToRowExec +(101) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(101) Scan parquet +(102) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(102) Filter +(103) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(103) Exchange +(104) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) Scan parquet +(105) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(105) Filter +(106) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(106) Exchange +(107) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(108) Project +(109) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(109) Exchange +(110) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(112) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(114) Project +(115) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(115) Exchange +(116) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) Scan parquet +(117) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(117) Filter +(118) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(118) Exchange +(119) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(120) Project +(121) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(121) Exchange +(122) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(123) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(123) Filter +(124) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(124) Exchange +(125) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(126) Project +(127) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(127) Exchange +(128) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(129) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(129) Filter +(130) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(130) Exchange +(131) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(132) Project +(133) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(133) HashAggregate +(134) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(134) Exchange +(135) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(135) HashAggregate +(136) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(136) Exchange +(137) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) Sort +(138) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(138) AdaptiveSparkPlan +(139) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt index 8c66866aa476..47886e292bf7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt @@ -1,165 +1,166 @@ == Physical Plan == -AdaptiveSparkPlan (192) +AdaptiveSparkPlan (193) +- == Final Plan == - VeloxColumnarToRowExec (140) - +- ^ SortExecTransformer (138) - +- ^ InputIteratorTransformer (137) - +- ShuffleQueryStage (135), Statistics(X) - +- ColumnarExchange (134) - +- VeloxAppendBatches (133) - +- ^ ProjectExecTransformer (131) - +- ^ RegularHashAggregateExecTransformer (130) - +- ^ InputIteratorTransformer (129) - +- ShuffleQueryStage (127), Statistics(X) - +- ColumnarExchange (126) - +- ^ ProjectExecTransformer (124) - +- ^ FlushableHashAggregateExecTransformer (123) - +- ^ ProjectExecTransformer (122) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) - :- ^ InputIteratorTransformer (111) - : +- ShuffleQueryStage (109), Statistics(X) - : +- ColumnarExchange (108) - : +- VeloxAppendBatches (107) - : +- ^ ProjectExecTransformer (105) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) - : :- ^ InputIteratorTransformer (94) - : : +- ShuffleQueryStage (92), Statistics(X) - : : +- ColumnarExchange (91) - : : +- VeloxAppendBatches (90) - : : +- ^ ProjectExecTransformer (88) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - : : :- ^ InputIteratorTransformer (77) - : : : +- ShuffleQueryStage (75), Statistics(X) - : : : +- ColumnarExchange (74) - : : : +- VeloxAppendBatches (73) - : : : +- ^ ProjectExecTransformer (71) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) - : : : :- ^ InputIteratorTransformer (60) - : : : : +- ShuffleQueryStage (58), Statistics(X) - : : : : +- ColumnarExchange (57) - : : : : +- VeloxAppendBatches (56) - : : : : +- ^ ProjectExecTransformer (54) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) - : : : : :- ^ InputIteratorTransformer (43) - : : : : : +- ShuffleQueryStage (41), Statistics(X) - : : : : : +- ColumnarExchange (40) - : : : : : +- VeloxAppendBatches (39) - : : : : : +- ^ ProjectExecTransformer (37) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : : : :- ^ InputIteratorTransformer (26) - : : : : : : +- ShuffleQueryStage (24), Statistics(X) - : : : : : : +- ColumnarExchange (23) - : : : : : : +- VeloxAppendBatches (22) - : : : : : : +- ^ ProjectExecTransformer (20) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : : : :- ^ InputIteratorTransformer (9) - : : : : : : : +- ShuffleQueryStage (7), Statistics(X) - : : : : : : : +- ColumnarExchange (6) - : : : : : : : +- VeloxAppendBatches (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (18) - : : : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : : : +- ColumnarExchange (15) - : : : : : : +- VeloxAppendBatches (14) - : : : : : : +- ^ ProjectExecTransformer (12) - : : : : : : +- ^ FilterExecTransformer (11) - : : : : : : +- ^ Scan parquet (10) - : : : : : +- ^ InputIteratorTransformer (35) - : : : : : +- ShuffleQueryStage (33), Statistics(X) - : : : : : +- ColumnarExchange (32) - : : : : : +- VeloxAppendBatches (31) - : : : : : +- ^ ProjectExecTransformer (29) - : : : : : +- ^ FilterExecTransformer (28) - : : : : : +- ^ Scan parquet (27) - : : : : +- ^ InputIteratorTransformer (52) - : : : : +- ShuffleQueryStage (50), Statistics(X) - : : : : +- ColumnarExchange (49) - : : : : +- VeloxAppendBatches (48) - : : : : +- ^ ProjectExecTransformer (46) - : : : : +- ^ FilterExecTransformer (45) - : : : : +- ^ Scan parquet (44) - : : : +- ^ InputIteratorTransformer (69) - : : : +- ShuffleQueryStage (67), Statistics(X) - : : : +- ColumnarExchange (66) - : : : +- VeloxAppendBatches (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ FilterExecTransformer (62) - : : : +- ^ Scan parquet (61) - : : +- ^ InputIteratorTransformer (86) - : : +- ShuffleQueryStage (84), Statistics(X) - : : +- ColumnarExchange (83) - : : +- VeloxAppendBatches (82) - : : +- ^ ProjectExecTransformer (80) - : : +- ^ FilterExecTransformer (79) - : : +- ^ Scan parquet (78) - : +- ^ InputIteratorTransformer (103) - : +- ShuffleQueryStage (101), Statistics(X) - : +- ColumnarExchange (100) - : +- VeloxAppendBatches (99) - : +- ^ ProjectExecTransformer (97) - : +- ^ FilterExecTransformer (96) - : +- ^ Scan parquet (95) - +- ^ InputIteratorTransformer (120) - +- ShuffleQueryStage (118), Statistics(X) - +- ColumnarExchange (117) - +- VeloxAppendBatches (116) - +- ^ ProjectExecTransformer (114) - +- ^ FilterExecTransformer (113) - +- ^ Scan parquet (112) + VeloxColumnarToRowExec (141) + +- ^ SortExecTransformer (139) + +- ^ InputIteratorTransformer (138) + +- ShuffleQueryStage (136), Statistics(X) + +- ColumnarExchange (135) + +- VeloxAppendBatches (134) + +- ^ ProjectExecTransformer (132) + +- ^ RegularHashAggregateExecTransformer (131) + +- ^ InputIteratorTransformer (130) + +- ShuffleQueryStage (128), Statistics(X) + +- ColumnarExchange (127) + +- VeloxAppendBatches (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109), Statistics(X) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92), Statistics(X) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75), Statistics(X) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58), Statistics(X) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41), Statistics(X) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ FilterExecTransformer (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33), Statistics(X) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ FilterExecTransformer (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50), Statistics(X) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ FilterExecTransformer (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67), Statistics(X) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ FilterExecTransformer (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ FilterExecTransformer (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ FilterExecTransformer (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118), Statistics(X) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ FilterExecTransformer (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (191) - +- Exchange (190) - +- HashAggregate (189) - +- Exchange (188) - +- HashAggregate (187) - +- Project (186) - +- ShuffledHashJoin Inner BuildRight (185) - :- Exchange (180) - : +- Project (179) - : +- ShuffledHashJoin Inner BuildRight (178) - : :- Exchange (174) - : : +- Project (173) - : : +- ShuffledHashJoin Inner BuildRight (172) - : : :- Exchange (168) - : : : +- Project (167) - : : : +- ShuffledHashJoin Inner BuildRight (166) - : : : :- Exchange (162) - : : : : +- Project (161) - : : : : +- ShuffledHashJoin Inner BuildRight (160) - : : : : :- Exchange (156) - : : : : : +- Project (155) - : : : : : +- ShuffledHashJoin Inner BuildRight (154) - : : : : : :- Exchange (150) - : : : : : : +- Project (149) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (148) - : : : : : : :- Exchange (144) - : : : : : : : +- Project (143) - : : : : : : : +- Filter (142) - : : : : : : : +- Scan parquet (141) - : : : : : : +- Exchange (147) - : : : : : : +- Filter (146) - : : : : : : +- Scan parquet (145) - : : : : : +- Exchange (153) - : : : : : +- Filter (152) - : : : : : +- Scan parquet (151) - : : : : +- Exchange (159) - : : : : +- Filter (158) - : : : : +- Scan parquet (157) - : : : +- Exchange (165) - : : : +- Filter (164) - : : : +- Scan parquet (163) - : : +- Exchange (171) - : : +- Filter (170) - : : +- Scan parquet (169) - : +- Exchange (177) - : +- Filter (176) - : +- Scan parquet (175) - +- Exchange (184) - +- Project (183) - +- Filter (182) - +- Scan parquet (181) + Sort (192) + +- Exchange (191) + +- HashAggregate (190) + +- Exchange (189) + +- HashAggregate (188) + +- Project (187) + +- ShuffledHashJoin Inner BuildRight (186) + :- Exchange (181) + : +- Project (180) + : +- ShuffledHashJoin Inner BuildRight (179) + : :- Exchange (175) + : : +- Project (174) + : : +- ShuffledHashJoin Inner BuildRight (173) + : : :- Exchange (169) + : : : +- Project (168) + : : : +- ShuffledHashJoin Inner BuildRight (167) + : : : :- Exchange (163) + : : : : +- Project (162) + : : : : +- ShuffledHashJoin Inner BuildRight (161) + : : : : :- Exchange (157) + : : : : : +- Project (156) + : : : : : +- ShuffledHashJoin Inner BuildRight (155) + : : : : : :- Exchange (151) + : : : : : : +- Project (150) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (149) + : : : : : : :- Exchange (145) + : : : : : : : +- Project (144) + : : : : : : : +- Filter (143) + : : : : : : : +- Scan parquet (142) + : : : : : : +- Exchange (148) + : : : : : : +- Filter (147) + : : : : : : +- Scan parquet (146) + : : : : : +- Exchange (154) + : : : : : +- Filter (153) + : : : : : +- Scan parquet (152) + : : : : +- Exchange (160) + : : : : +- Filter (159) + : : : : +- Scan parquet (158) + : : : +- Exchange (166) + : : : +- Filter (165) + : : : +- Scan parquet (164) + : : +- Exchange (172) + : : +- Filter (171) + : : +- Scan parquet (170) + : +- Exchange (178) + : +- Filter (177) + : +- Scan parquet (176) + +- Exchange (185) + +- Project (184) + +- Filter (183) + +- Scan parquet (182) (1) Scan parquet @@ -668,305 +669,309 @@ Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(126) ColumnarExchange +(126) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(127) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(127) ShuffleQueryStage +(128) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(128) InputAdapter +(129) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(129) InputIteratorTransformer +(130) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(130) RegularHashAggregateExecTransformer +(131) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(131) ProjectExecTransformer +(132) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(132) WholeStageCodegenTransformer (X) +(133) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(133) VeloxAppendBatches +(134) VeloxAppendBatches Input [2]: [o_year#X, mkt_share#X] Arguments: X -(134) ColumnarExchange +(135) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(135) ShuffleQueryStage +(136) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(136) InputAdapter +(137) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(137) InputIteratorTransformer +(138) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(138) SortExecTransformer +(139) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(139) WholeStageCodegenTransformer (X) +(140) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(140) VeloxColumnarToRowExec +(141) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(141) Scan parquet +(142) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(142) Filter +(143) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(143) Project +(144) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(144) Exchange +(145) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) Scan parquet +(146) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(146) Filter +(147) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(147) Exchange +(148) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) ShuffledHashJoin +(149) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(149) Project +(150) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(150) Exchange +(151) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) Scan parquet +(152) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(152) Filter +(153) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(153) Exchange +(154) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) ShuffledHashJoin +(155) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(155) Project +(156) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(156) Exchange +(157) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) Scan parquet +(158) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(158) Filter +(159) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(159) Exchange +(160) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) ShuffledHashJoin +(161) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(161) Project +(162) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(162) Exchange +(163) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) Scan parquet +(164) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(164) Filter +(165) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(165) Exchange +(166) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) ShuffledHashJoin +(167) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(167) Project +(168) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(168) Exchange +(169) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(169) Scan parquet +(170) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(170) Filter +(171) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(171) Exchange +(172) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(172) ShuffledHashJoin +(173) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(173) Project +(174) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(174) Exchange +(175) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(175) Scan parquet +(176) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(176) Filter +(177) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(177) Exchange +(178) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(178) ShuffledHashJoin +(179) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(179) Project +(180) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(180) Exchange +(181) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(181) Scan parquet +(182) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(182) Filter +(183) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(183) Project +(184) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(184) Exchange +(185) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(185) ShuffledHashJoin +(186) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(186) Project +(187) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(187) HashAggregate +(188) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(188) Exchange +(189) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(189) HashAggregate +(190) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(190) Exchange +(191) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(191) Sort +(192) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(192) AdaptiveSparkPlan +(193) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt index 7435043d769d..0bad5c20cf05 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt @@ -1,125 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (144) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (105) - +- ^ SortExecTransformer (103) - +- ^ InputIteratorTransformer (102) - +- ShuffleQueryStage (100), Statistics(X) - +- ColumnarExchange (99) - +- VeloxAppendBatches (98) - +- ^ RegularHashAggregateExecTransformer (96) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ColumnarExchange (92) - +- ^ ProjectExecTransformer (90) - +- ^ FlushableHashAggregateExecTransformer (89) - +- ^ ProjectExecTransformer (88) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - :- ^ InputIteratorTransformer (77) - : +- ShuffleQueryStage (75), Statistics(X) - : +- ColumnarExchange (74) - : +- VeloxAppendBatches (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) - : :- ^ InputIteratorTransformer (60) - : : +- ShuffleQueryStage (58), Statistics(X) - : : +- ColumnarExchange (57) - : : +- VeloxAppendBatches (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) - : : :- ^ InputIteratorTransformer (43) - : : : +- ShuffleQueryStage (41), Statistics(X) - : : : +- ColumnarExchange (40) - : : : +- VeloxAppendBatches (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : :- ^ InputIteratorTransformer (26) - : : : : +- ShuffleQueryStage (24), Statistics(X) - : : : : +- ColumnarExchange (23) - : : : : +- VeloxAppendBatches (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ InputIteratorTransformer (9) - : : : : : +- ShuffleQueryStage (7), Statistics(X) - : : : : : +- ColumnarExchange (6) - : : : : : +- VeloxAppendBatches (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ FilterExecTransformer (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (35) - : : : +- ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- VeloxAppendBatches (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ FilterExecTransformer (28) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50), Statistics(X) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ FilterExecTransformer (45) - : : +- ^ Scan parquet (44) - : +- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67), Statistics(X) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ FilterExecTransformer (62) - : +- ^ Scan parquet (61) - +- ^ InputIteratorTransformer (86) - +- ShuffleQueryStage (84), Statistics(X) - +- ColumnarExchange (83) - +- VeloxAppendBatches (82) - +- ^ ProjectExecTransformer (80) - +- ^ FilterExecTransformer (79) - +- ^ Scan parquet (78) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94), Statistics(X) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (143) - +- Exchange (142) - +- HashAggregate (141) - +- Exchange (140) - +- HashAggregate (139) - +- Project (138) - +- ShuffledHashJoin Inner BuildRight (137) - :- Exchange (133) - : +- Project (132) - : +- ShuffledHashJoin Inner BuildRight (131) - : :- Exchange (127) - : : +- Project (126) - : : +- ShuffledHashJoin Inner BuildRight (125) - : : :- Exchange (121) - : : : +- Project (120) - : : : +- ShuffledHashJoin Inner BuildRight (119) - : : : :- Exchange (115) - : : : : +- Project (114) - : : : : +- ShuffledHashJoin Inner BuildLeft (113) - : : : : :- Exchange (109) - : : : : : +- Project (108) - : : : : : +- Filter (107) - : : : : : +- Scan parquet (106) - : : : : +- Exchange (112) - : : : : +- Filter (111) - : : : : +- Scan parquet (110) - : : : +- Exchange (118) - : : : +- Filter (117) - : : : +- Scan parquet (116) - : : +- Exchange (124) - : : +- Filter (123) - : : +- Scan parquet (122) - : +- Exchange (130) - : +- Filter (129) - : +- Scan parquet (128) - +- Exchange (136) - +- Filter (135) - +- Scan parquet (134) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (110) + : : : : : +- Project (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Filter (112) + : : : : +- Scan parquet (111) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -492,241 +493,245 @@ Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(92) ColumnarExchange +(92) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(93) ShuffleQueryStage +(94) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(94) InputAdapter +(95) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(95) InputIteratorTransformer +(96) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(96) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(97) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(98) VeloxAppendBatches +(99) VeloxAppendBatches Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(99) ColumnarExchange +(100) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(100) ShuffleQueryStage +(101) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(101) InputAdapter +(102) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(102) InputIteratorTransformer +(103) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(103) SortExecTransformer +(104) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(104) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(105) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(106) Scan parquet +(107) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(107) Filter +(108) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(108) Project +(109) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(109) Exchange +(110) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(111) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(112) Exchange +(113) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(114) Project +(115) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(115) Exchange +(116) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) Scan parquet +(117) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(117) Filter +(118) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(118) Exchange +(119) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(120) Project +(121) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(121) Exchange +(122) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(123) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(123) Filter +(124) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(124) Exchange +(125) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(126) Project +(127) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(127) Exchange +(128) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(129) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(129) Filter +(130) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(130) Exchange +(131) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(132) Project +(133) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(133) Exchange +(134) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(134) Scan parquet +(135) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(135) Filter +(136) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(136) Exchange +(137) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(138) Project +(139) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(139) HashAggregate +(140) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(140) Exchange +(141) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(141) HashAggregate +(142) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(142) Exchange +(143) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(143) Sort +(144) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(144) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/1.txt index 468ad7c5d3aa..1e53cd90e1b3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (29) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (28) - +- Exchange (27) - +- HashAggregate (26) - +- Exchange (25) - +- HashAggregate (24) - +- Project (23) - +- Filter (22) - +- Scan parquet (21) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -57,101 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) VeloxAppendBatches +(14) VeloxAppendBatches Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) ColumnarExchange +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(18) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(20) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(21) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(22) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(23) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(24) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(25) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(27) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(29) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt index c740b49a1a8b..5be72ee42483 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt @@ -1,84 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (66) - +- TakeOrderedAndProjectExecTransformer (65) - +- ^ ProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ ProjectExecTransformer (56) - +- ^ FlushableHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - :- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41), Statistics(X) - : +- ColumnarExchange (40) - : +- VeloxAppendBatches (39) - : +- ^ ProjectExecTransformer (37) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - : :- ^ InputIteratorTransformer (26) - : : +- ShuffleQueryStage (24), Statistics(X) - : : +- ColumnarExchange (23) - : : +- VeloxAppendBatches (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : : :- ^ InputIteratorTransformer (9) - : : : +- ShuffleQueryStage (7), Statistics(X) - : : : +- ColumnarExchange (6) - : : : +- VeloxAppendBatches (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (18) - : : +- ShuffleQueryStage (16), Statistics(X) - : : +- ColumnarExchange (15) - : : +- VeloxAppendBatches (14) - : : +- ^ ProjectExecTransformer (12) - : : +- ^ FilterExecTransformer (11) - : : +- ^ Scan parquet (10) - : +- ^ InputIteratorTransformer (35) - : +- ShuffleQueryStage (33), Statistics(X) - : +- ColumnarExchange (32) - : +- VeloxAppendBatches (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ FilterExecTransformer (28) - : +- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- VeloxAppendBatches (48) - +- ^ ProjectExecTransformer (46) - +- ^ FilterExecTransformer (45) - +- ^ Scan parquet (44) + VeloxColumnarToRowExec (67) + +- TakeOrderedAndProjectExecTransformer (66) + +- ^ ProjectExecTransformer (64) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24), Statistics(X) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33), Statistics(X) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FilterExecTransformer (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (92) - +- HashAggregate (91) - +- Exchange (90) - +- HashAggregate (89) - +- Project (88) - +- ShuffledHashJoin Inner BuildRight (87) - :- Exchange (83) - : +- Project (82) - : +- ShuffledHashJoin Inner BuildRight (81) - : :- Exchange (76) - : : +- Project (75) - : : +- ShuffledHashJoin Inner BuildRight (74) - : : :- Exchange (69) - : : : +- Filter (68) - : : : +- Scan parquet (67) - : : +- Exchange (73) - : : +- Project (72) - : : +- Filter (71) - : : +- Scan parquet (70) - : +- Exchange (80) - : +- Project (79) - : +- Filter (78) - : +- Scan parquet (77) - +- Exchange (86) - +- Filter (85) - +- Scan parquet (84) + TakeOrderedAndProject (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- ShuffledHashJoin Inner BuildRight (88) + :- Exchange (84) + : +- Project (83) + : +- ShuffledHashJoin Inner BuildRight (82) + : :- Exchange (77) + : : +- Project (76) + : : +- ShuffledHashJoin Inner BuildRight (75) + : : :- Exchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- Exchange (74) + : : +- Project (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (81) + : +- Project (80) + : +- Filter (79) + : +- Scan parquet (78) + +- Exchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -318,170 +319,174 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(58) ColumnarExchange +(58) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(59) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(60) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(60) InputAdapter +(61) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(61) InputIteratorTransformer +(62) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(62) RegularHashAggregateExecTransformer +(63) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(63) ProjectExecTransformer +(64) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(65) TakeOrderedAndProjectExecTransformer +(66) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(66) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) Scan parquet +(68) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(68) Filter +(69) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(69) Exchange +(70) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(70) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(72) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(73) Exchange +(74) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) ShuffledHashJoin +(75) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(75) Project +(76) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(76) Exchange +(77) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Scan parquet +(78) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(78) Filter +(79) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(79) Project +(80) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(80) Exchange +(81) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(82) Project +(83) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(83) Exchange +(84) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(86) Exchange +(87) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) ShuffledHashJoin +(88) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(88) Project +(89) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(89) HashAggregate +(90) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(90) Exchange +(91) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(92) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(92) TakeOrderedAndProject +(93) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(93) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt index 89dd19ffe0ce..d9dbbfe0dbe9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt @@ -1,70 +1,71 @@ == Physical Plan == -AdaptiveSparkPlan (77) +AdaptiveSparkPlan (78) +- == Final Plan == - VeloxColumnarToRowExec (55) - +- ^ SortExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- VeloxAppendBatches (48) - +- ^ FilterExecTransformer (46) - +- ^ RegularHashAggregateExecTransformer (45) - +- ^ InputIteratorTransformer (44) - +- ShuffleQueryStage (42), Statistics(X) - +- ColumnarExchange (41) - +- ^ ProjectExecTransformer (39) - +- ^ FlushableHashAggregateExecTransformer (38) - +- ^ ProjectExecTransformer (37) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - :- ^ InputIteratorTransformer (26) - : +- ShuffleQueryStage (24), Statistics(X) - : +- ColumnarExchange (23) - : +- VeloxAppendBatches (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : :- ^ InputIteratorTransformer (9) - : : +- ShuffleQueryStage (7), Statistics(X) - : : +- ColumnarExchange (6) - : : +- VeloxAppendBatches (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (18) - : +- ShuffleQueryStage (16), Statistics(X) - : +- ColumnarExchange (15) - : +- VeloxAppendBatches (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FilterExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- VeloxAppendBatches (31) - +- ^ ProjectExecTransformer (29) - +- ^ FilterExecTransformer (28) - +- ^ Scan parquet (27) + VeloxColumnarToRowExec (56) + +- ^ SortExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ShuffleQueryStage (51), Statistics(X) + +- ColumnarExchange (50) + +- VeloxAppendBatches (49) + +- ^ FilterExecTransformer (47) + +- ^ RegularHashAggregateExecTransformer (46) + +- ^ InputIteratorTransformer (45) + +- ShuffleQueryStage (43), Statistics(X) + +- ColumnarExchange (42) + +- VeloxAppendBatches (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (76) - +- Exchange (75) - +- Filter (74) - +- HashAggregate (73) - +- Exchange (72) - +- HashAggregate (71) - +- Project (70) - +- ShuffledHashJoin Inner BuildRight (69) - :- Exchange (64) - : +- Project (63) - : +- ShuffledHashJoin Inner BuildRight (62) - : :- Exchange (58) - : : +- Filter (57) - : : +- Scan parquet (56) - : +- Exchange (61) - : +- Filter (60) - : +- Scan parquet (59) - +- Exchange (68) - +- Project (67) - +- Filter (66) - +- Scan parquet (65) + Sort (77) + +- Exchange (76) + +- Filter (75) + +- HashAggregate (74) + +- Exchange (73) + +- HashAggregate (72) + +- Project (71) + +- ShuffledHashJoin Inner BuildRight (70) + :- Exchange (65) + : +- Project (64) + : +- ShuffledHashJoin Inner BuildRight (63) + : :- Exchange (59) + : : +- Filter (58) + : : +- Scan parquet (57) + : +- Exchange (62) + : +- Filter (61) + : +- Scan parquet (60) + +- Exchange (69) + +- Project (68) + +- Filter (67) + +- Scan parquet (66) (1) Scan parquet @@ -235,438 +236,442 @@ Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(41) ColumnarExchange +(41) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(42) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(42) ShuffleQueryStage +(43) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(43) InputAdapter +(44) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(44) InputIteratorTransformer +(45) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(45) RegularHashAggregateExecTransformer +(46) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(46) FilterExecTransformer +(47) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(47) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(48) VeloxAppendBatches +(49) VeloxAppendBatches Input [2]: [ps_partkey#X, value#X] Arguments: X -(49) ColumnarExchange +(50) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(51) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(51) InputAdapter +(52) InputAdapter Input [2]: [ps_partkey#X, value#X] -(52) InputIteratorTransformer +(53) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(53) SortExecTransformer +(54) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(54) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(55) VeloxColumnarToRowExec +(56) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(56) Scan parquet +(57) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(57) Filter +(58) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(58) Exchange +(59) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) Scan parquet +(60) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(60) Filter +(61) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(61) Exchange +(62) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) ShuffledHashJoin +(63) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(63) Project +(64) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(64) Exchange +(65) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) Scan parquet +(66) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(67) Project +(68) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(68) Exchange +(69) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(69) ShuffledHashJoin +(70) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(70) Project +(71) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(71) HashAggregate +(72) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(72) Exchange +(73) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(73) HashAggregate +(74) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(74) Filter +(75) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(75) Exchange +(76) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) Sort +(77) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(77) AdaptiveSparkPlan +(78) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (127) +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (109) - +- ^ ProjectExecTransformer (107) - +- ^ RegularHashAggregateExecTransformer (106) - +- ^ RegularHashAggregateExecTransformer (105) - +- ^ ProjectExecTransformer (104) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (103) - :- ^ InputIteratorTransformer (98) - : +- ShuffleQueryStage (96), Statistics(X) - : +- ColumnarExchange (95) - : +- VeloxAppendBatches (94) - : +- ^ ProjectExecTransformer (92) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (91) - : :- ^ InputIteratorTransformer (86) - : : +- ShuffleQueryStage (84), Statistics(X) - : : +- ColumnarExchange (83) - : : +- VeloxAppendBatches (82) - : : +- ^ ProjectExecTransformer (80) - : : +- ^ FilterExecTransformer (79) - : : +- ^ Scan parquet (78) - : +- ^ InputIteratorTransformer (90) - : +- ShuffleQueryStage (88), Statistics(X) - : +- ReusedExchange (87) - +- ^ InputIteratorTransformer (102) - +- ShuffleQueryStage (100), Statistics(X) - +- ReusedExchange (99) + VeloxColumnarToRowExec (110) + +- ^ ProjectExecTransformer (108) + +- ^ RegularHashAggregateExecTransformer (107) + +- ^ RegularHashAggregateExecTransformer (106) + +- ^ ProjectExecTransformer (105) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + :- ^ InputIteratorTransformer (99) + : +- ShuffleQueryStage (97), Statistics(X) + : +- ColumnarExchange (96) + : +- VeloxAppendBatches (95) + : +- ^ ProjectExecTransformer (93) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) + : :- ^ InputIteratorTransformer (87) + : : +- ShuffleQueryStage (85), Statistics(X) + : : +- ColumnarExchange (84) + : : +- VeloxAppendBatches (83) + : : +- ^ ProjectExecTransformer (81) + : : +- ^ FilterExecTransformer (80) + : : +- ^ Scan parquet (79) + : +- ^ InputIteratorTransformer (91) + : +- ShuffleQueryStage (89), Statistics(X) + : +- ReusedExchange (88) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ReusedExchange (100) +- == Initial Plan == - HashAggregate (126) - +- HashAggregate (125) - +- Project (124) - +- ShuffledHashJoin Inner BuildRight (123) - :- Exchange (118) - : +- Project (117) - : +- ShuffledHashJoin Inner BuildRight (116) - : :- Exchange (112) - : : +- Filter (111) - : : +- Scan parquet (110) - : +- Exchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- Exchange (122) - +- Project (121) - +- Filter (120) - +- Scan parquet (119) - - -(78) Scan parquet + HashAggregate (127) + +- HashAggregate (126) + +- Project (125) + +- ShuffledHashJoin Inner BuildRight (124) + :- Exchange (119) + : +- Project (118) + : +- ShuffledHashJoin Inner BuildRight (117) + : :- Exchange (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Exchange (116) + : +- Filter (115) + : +- Scan parquet (114) + +- Exchange (123) + +- Project (122) + +- Filter (121) + +- Scan parquet (120) + + +(79) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(79) FilterExecTransformer +(80) FilterExecTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: isnotnull(ps_suppkey#X) -(80) ProjectExecTransformer +(81) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(81) WholeStageCodegenTransformer (X) +(82) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(82) VeloxAppendBatches +(83) VeloxAppendBatches Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(83) ColumnarExchange +(84) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(84) ShuffleQueryStage +(85) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(85) InputAdapter +(86) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(86) InputIteratorTransformer +(87) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(87) ReusedExchange [Reuses operator id: 15] +(88) ReusedExchange [Reuses operator id: 15] Output [2]: [s_suppkey#X, s_nationkey#X] -(88) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(89) InputAdapter +(90) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(90) InputIteratorTransformer +(91) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(91) ShuffledHashJoinExecTransformer +(92) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(92) ProjectExecTransformer +(93) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(93) WholeStageCodegenTransformer (X) +(94) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(94) VeloxAppendBatches +(95) VeloxAppendBatches Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(95) ColumnarExchange +(96) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(97) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(97) InputAdapter +(98) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(98) InputIteratorTransformer +(99) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(99) ReusedExchange [Reuses operator id: 32] +(100) ReusedExchange [Reuses operator id: 32] Output [1]: [n_nationkey#X] -(100) ShuffleQueryStage +(101) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(101) InputAdapter +(102) InputAdapter Input [1]: [n_nationkey#X] -(102) InputIteratorTransformer +(103) InputIteratorTransformer Input [1]: [n_nationkey#X] -(103) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(104) ProjectExecTransformer +(105) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(105) RegularHashAggregateExecTransformer +(106) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(106) RegularHashAggregateExecTransformer +(107) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(107) ProjectExecTransformer +(108) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(108) WholeStageCodegenTransformer (X) +(109) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(109) VeloxColumnarToRowExec +(110) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(110) Scan parquet +(111) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(112) Exchange +(113) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) Scan parquet +(114) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(114) Filter +(115) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(115) Exchange +(116) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(117) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(117) Project +(118) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(118) Exchange +(119) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) Scan parquet +(120) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(120) Filter +(121) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(121) Project +(122) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(122) Exchange +(123) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) ShuffledHashJoin +(124) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(124) Project +(125) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(125) HashAggregate +(126) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(126) HashAggregate +(127) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(127) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt index 75948ca7ad8b..63c356d6d1bf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt @@ -1,49 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32), Statistics(X) - +- ColumnarExchange (31) - +- VeloxAppendBatches (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7), Statistics(X) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- VeloxAppendBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ FilterExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- Project (46) - +- ShuffledHashJoin Inner BuildLeft (45) - :- Exchange (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Exchange (44) - +- Project (43) - +- Filter (42) - +- Scan parquet (41) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin Inner BuildLeft (46) + :- Exchange (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -145,130 +146,134 @@ Input [3]: [l_shipmode#X, sum#X, sum#X] Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(24) ColumnarExchange +(24) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(25) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(28) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(29) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(30) VeloxAppendBatches +(31) VeloxAppendBatches Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(31) ColumnarExchange +(32) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) SortExecTransformer +(36) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(37) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(38) Scan parquet +(39) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(40) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(42) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(42) Filter +(43) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(43) Project +(44) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(44) Exchange +(45) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(46) Project +(47) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(47) HashAggregate +(48) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(48) Exchange +(49) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(50) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(50) Exchange +(51) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(52) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(52) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt index 5a8bd7bbc955..812a7be868b6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (55) +AdaptiveSparkPlan (56) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- ^ SortExecTransformer (37) - +- ^ InputIteratorTransformer (36) - +- ShuffleQueryStage (34), Statistics(X) - +- ColumnarExchange (33) - +- VeloxAppendBatches (32) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- VeloxAppendBatches (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (54) - +- Exchange (53) - +- HashAggregate (52) - +- Exchange (51) - +- HashAggregate (50) - +- HashAggregate (49) - +- HashAggregate (48) - +- Project (47) - +- ShuffledHashJoin LeftOuter BuildRight (46) - :- Exchange (41) - : +- Scan parquet (40) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftOuter BuildRight (47) + :- Exchange (42) + : +- Scan parquet (41) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -161,139 +162,143 @@ Input [2]: [c_count#X, count#X] Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [2]: [c_count#X, count#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(32) VeloxAppendBatches +(33) VeloxAppendBatches Input [2]: [c_count#X, custdist#X] Arguments: X -(33) ColumnarExchange +(34) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(34) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(35) InputAdapter +(36) InputAdapter Input [2]: [c_count#X, custdist#X] -(36) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(37) SortExecTransformer +(38) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(38) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(39) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(40) Scan parquet +(41) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(41) Exchange +(42) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Scan parquet +(43) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(43) Filter +(44) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(44) Project +(45) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(45) Exchange +(46) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(47) Project +(48) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(48) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(49) HashAggregate +(50) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(50) HashAggregate +(51) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(51) Exchange +(52) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) HashAggregate +(53) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(53) Exchange +(54) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Sort +(55) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(55) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt index 5592b9e73add..e30eec3d854f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt @@ -1,13 +1,13 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- AQEShuffleRead (28) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- VeloxAppendBatches (25) - +- ^ ProjectExecTransformer (23) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (22) + VeloxColumnarToRowExec (30) + +- AQEShuffleRead (29) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (23) :- ^ InputIteratorTransformer (9) : +- ShuffleQueryStage (7), Statistics(X) : +- ColumnarExchange (6) @@ -15,31 +15,32 @@ AdaptiveSparkPlan (44) : +- ^ ProjectExecTransformer (3) : +- ^ FilterExecTransformer (2) : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ FilterExecTransformer (11) - +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- Project (41) - +- ShuffledHashJoin Inner BuildLeft (40) - :- Exchange (32) - : +- Filter (31) - : +- Scan parquet (30) - +- Filter (39) - +- HashAggregate (38) - +- Exchange (37) - +- HashAggregate (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (44) + +- Exchange (43) + +- Project (42) + +- ShuffledHashJoin Inner BuildLeft (41) + :- Exchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Filter (40) + +- HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -109,290 +110,299 @@ Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(16) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(21) FilterExecTransformer +(22) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(22) ShuffledHashJoinExecTransformer +(23) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(23) ProjectExecTransformer +(24) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(24) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(25) VeloxAppendBatches +(26) VeloxAppendBatches Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) ColumnarExchange +(27) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(28) AQEShuffleRead +(29) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(30) Scan parquet +(31) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(32) Exchange +(33) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(33) Scan parquet +(34) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(34) Filter +(35) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(35) Project +(36) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(36) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(37) Exchange +(38) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) HashAggregate +(39) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(39) Filter +(40) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(40) ShuffledHashJoin +(41) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(41) Project +(42) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(42) Exchange +(43) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(44) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 21 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (69) +Subquery:1 Hosting operator id = 22 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (71) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ RegularHashAggregateExecTransformer (58) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ ProjectExecTransformer (56) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ InputIteratorTransformer (54) - +- ShuffleQueryStage (52), Statistics(X) - +- ColumnarExchange (51) - +- ^ ProjectExecTransformer (49) - +- ^ FlushableHashAggregateExecTransformer (48) - +- ^ ProjectExecTransformer (47) - +- ^ FilterExecTransformer (46) - +- ^ Scan parquet (45) + VeloxColumnarToRowExec (62) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ ProjectExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ FilterExecTransformer (47) + +- ^ Scan parquet (46) +- == Initial Plan == - HashAggregate (68) - +- HashAggregate (67) - +- HashAggregate (66) - +- Exchange (65) - +- HashAggregate (64) - +- Project (63) - +- Filter (62) - +- Scan parquet (61) + HashAggregate (70) + +- HashAggregate (69) + +- HashAggregate (68) + +- Exchange (67) + +- HashAggregate (66) + +- Project (65) + +- Filter (64) + +- Scan parquet (63) -(45) Scan parquet +(46) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(46) FilterExecTransformer +(47) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(47) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(48) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(49) ProjectExecTransformer +(50) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(50) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(51) ColumnarExchange +(52) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(53) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(54) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(53) InputAdapter +(55) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(54) InputIteratorTransformer +(56) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(55) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(56) ProjectExecTransformer +(58) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(57) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(58) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(59) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(60) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(61) Scan parquet +(63) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(62) Filter +(64) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(63) Project +(65) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(64) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(65) Exchange +(67) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) HashAggregate +(68) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(67) HashAggregate +(69) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(68) HashAggregate +(70) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(69) AdaptiveSparkPlan +(71) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt index 73703d5baf67..eff7577281e6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt @@ -1,62 +1,64 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (45) - +- ^ SortExecTransformer (43) - +- ^ InputIteratorTransformer (42) - +- ShuffleQueryStage (40), Statistics(X) - +- ColumnarExchange (39) - +- VeloxAppendBatches (38) - +- ^ RegularHashAggregateExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ ProjectExecTransformer (30) - +- ^ FlushableHashAggregateExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7), Statistics(X) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- VeloxAppendBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ FilterExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (47) + +- ^ SortExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42), Statistics(X) + +- ColumnarExchange (41) + +- VeloxAppendBatches (40) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ ProjectExecTransformer (31) + +- ^ FlushableHashAggregateExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (66) - +- Exchange (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- Project (58) - +- ShuffledHashJoin Inner BuildRight (57) - :- Exchange (53) - : +- BroadcastHashJoin LeftAnti BuildRight (52) - : :- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (51) - : +- Project (50) - : +- Filter (49) - : +- Scan parquet (48) - +- Exchange (56) - +- Filter (55) - +- Scan parquet (54) + Sort (68) + +- Exchange (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- HashAggregate (63) + +- Exchange (62) + +- HashAggregate (61) + +- Project (60) + +- ShuffledHashJoin Inner BuildRight (59) + :- Exchange (55) + : +- BroadcastHashJoin LeftAnti BuildRight (54) + : :- Filter (49) + : : +- Scan parquet (48) + : +- BroadcastExchange (53) + : +- Project (52) + : +- Filter (51) + : +- Scan parquet (50) + +- Exchange (58) + +- Filter (57) + +- Scan parquet (56) (1) Scan parquet @@ -158,205 +160,213 @@ Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(24) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(28) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(29) FlushableHashAggregateExecTransformer +(30) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(30) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(32) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(34) InputAdapter +(36) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(35) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(36) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(37) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(38) VeloxAppendBatches +(40) VeloxAppendBatches Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(39) ColumnarExchange +(41) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(42) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(41) InputAdapter +(43) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(42) InputIteratorTransformer +(44) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) SortExecTransformer +(45) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(44) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(45) VeloxColumnarToRowExec +(47) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(46) Scan parquet +(48) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(47) Filter +(49) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(48) Scan parquet +(50) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(49) Filter +(51) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(50) Project +(52) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(51) BroadcastExchange +(53) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(52) BroadcastHashJoin +(54) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: LeftAnti Join condition: None -(53) Exchange +(55) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(56) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(56) Exchange +(58) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(59) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(58) Project +(60) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(59) HashAggregate +(61) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(60) Exchange +(62) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(63) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(62) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(63) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(66) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(65) Exchange +(67) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) Sort +(68) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(67) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt index 081d1e972014..649bfcbe40e1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt @@ -1,12 +1,12 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- ^ ProjectExecTransformer (37) - +- ^ RegularHashAggregateExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ ProjectExecTransformer (34) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (33) + VeloxColumnarToRowExec (40) + +- ^ ProjectExecTransformer (38) + +- ^ RegularHashAggregateExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ ProjectExecTransformer (35) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (34) :- ^ ProjectExecTransformer (20) : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) : :- ^ InputIteratorTransformer (9) @@ -23,36 +23,37 @@ AdaptiveSparkPlan (59) : +- ^ ProjectExecTransformer (12) : +- ^ FilterExecTransformer (11) : +- ^ Scan parquet (10) - +- ^ FilterExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ FilterExecTransformer (22) - +- ^ Scan parquet (21) + +- ^ FilterExecTransformer (33) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ FilterExecTransformer (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (58) - +- HashAggregate (57) - +- Project (56) - +- ShuffledHashJoin Inner BuildRight (55) - :- Project (48) - : +- ShuffledHashJoin Inner BuildRight (47) - : :- Exchange (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Project (45) - : +- Filter (44) - : +- Scan parquet (43) - +- Filter (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- Filter (50) - +- Scan parquet (49) + HashAggregate (59) + +- HashAggregate (58) + +- Project (57) + +- ShuffledHashJoin Inner BuildRight (56) + :- Project (49) + : +- ShuffledHashJoin Inner BuildRight (48) + : :- Exchange (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Exchange (47) + : +- Project (46) + : +- Filter (45) + : +- Scan parquet (44) + +- Filter (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- Filter (51) + +- Scan parquet (50) (1) Scan parquet @@ -165,171 +166,175 @@ Input [3]: [l_partkey#X, sum#X, count#X] Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(32) FilterExecTransformer +(33) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(33) ShuffledHashJoinExecTransformer +(34) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(34) ProjectExecTransformer +(35) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(35) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(36) RegularHashAggregateExecTransformer +(37) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(37) ProjectExecTransformer +(38) ProjectExecTransformer Output [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(38) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(39) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(40) Scan parquet +(41) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(41) Filter +(42) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(42) Exchange +(43) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Scan parquet +(44) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(45) Project +(46) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(46) Exchange +(47) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(48) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(48) Project +(49) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(49) Scan parquet +(50) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(50) Filter +(51) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(51) HashAggregate +(52) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(52) Exchange +(53) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(54) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(54) Filter +(55) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(55) ShuffledHashJoin +(56) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(56) Project +(57) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(57) HashAggregate +(58) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(58) HashAggregate +(59) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] -(59) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt index 8c55d747d0ae..c3075e511782 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt @@ -1,18 +1,18 @@ == Physical Plan == -AdaptiveSparkPlan (102) +AdaptiveSparkPlan (103) +- == Final Plan == - VeloxColumnarToRowExec (69) - +- TakeOrderedAndProjectExecTransformer (68) - +- ^ RegularHashAggregateExecTransformer (66) - +- ^ RegularHashAggregateExecTransformer (65) - +- ^ ProjectExecTransformer (64) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (63) - :- ^ InputIteratorTransformer (45) - : +- ShuffleQueryStage (43), Statistics(X) - : +- ColumnarExchange (42) - : +- VeloxAppendBatches (41) - : +- ^ ProjectExecTransformer (39) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (38) + VeloxColumnarToRowExec (70) + +- TakeOrderedAndProjectExecTransformer (69) + +- ^ RegularHashAggregateExecTransformer (67) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ ProjectExecTransformer (65) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (64) + :- ^ InputIteratorTransformer (46) + : +- ShuffleQueryStage (44), Statistics(X) + : +- ColumnarExchange (43) + : +- VeloxAppendBatches (42) + : +- ^ ProjectExecTransformer (40) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (39) : :- ^ InputIteratorTransformer (9) : : +- ShuffleQueryStage (7), Statistics(X) : : +- ColumnarExchange (6) @@ -20,12 +20,12 @@ AdaptiveSparkPlan (102) : : +- ^ ProjectExecTransformer (3) : : +- ^ FilterExecTransformer (2) : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (37) - : +- ShuffleQueryStage (35), Statistics(X) - : +- ColumnarExchange (34) - : +- VeloxAppendBatches (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (30) + : +- ^ InputIteratorTransformer (38) + : +- ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- VeloxAppendBatches (34) + : +- ^ ProjectExecTransformer (32) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (31) : :- ^ InputIteratorTransformer (18) : : +- ShuffleQueryStage (16), Statistics(X) : : +- ColumnarExchange (15) @@ -33,62 +33,63 @@ AdaptiveSparkPlan (102) : : +- ^ ProjectExecTransformer (12) : : +- ^ FilterExecTransformer (11) : : +- ^ Scan parquet (10) - : +- ^ ProjectExecTransformer (29) - : +- ^ FilterExecTransformer (28) - : +- ^ RegularHashAggregateExecTransformer (27) - : +- ^ InputIteratorTransformer (26) - : +- ShuffleQueryStage (24), Statistics(X) - : +- ColumnarExchange (23) - : +- ^ ProjectExecTransformer (21) - : +- ^ FlushableHashAggregateExecTransformer (20) - : +- ^ Scan parquet (19) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - :- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- VeloxAppendBatches (50) - : +- ^ ProjectExecTransformer (48) - : +- ^ FilterExecTransformer (47) - : +- ^ Scan parquet (46) - +- ^ ProjectExecTransformer (61) - +- ^ FilterExecTransformer (60) - +- ^ RegularHashAggregateExecTransformer (59) - +- ^ InputIteratorTransformer (58) - +- ShuffleQueryStage (56), Statistics(X) - +- ReusedExchange (55) + : +- ^ ProjectExecTransformer (30) + : +- ^ FilterExecTransformer (29) + : +- ^ RegularHashAggregateExecTransformer (28) + : +- ^ InputIteratorTransformer (27) + : +- ShuffleQueryStage (25), Statistics(X) + : +- ColumnarExchange (24) + : +- VeloxAppendBatches (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (63) + :- ^ InputIteratorTransformer (55) + : +- ShuffleQueryStage (53), Statistics(X) + : +- ColumnarExchange (52) + : +- VeloxAppendBatches (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ Scan parquet (47) + +- ^ ProjectExecTransformer (62) + +- ^ FilterExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57), Statistics(X) + +- ReusedExchange (56) +- == Initial Plan == - TakeOrderedAndProject (101) - +- HashAggregate (100) - +- HashAggregate (99) - +- Project (98) - +- ShuffledHashJoin Inner BuildRight (97) - :- Exchange (86) - : +- Project (85) - : +- ShuffledHashJoin Inner BuildLeft (84) - : :- Exchange (72) - : : +- Filter (71) - : : +- Scan parquet (70) - : +- Exchange (83) - : +- ShuffledHashJoin LeftSemi BuildRight (82) - : :- Exchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- Project (81) - : +- Filter (80) - : +- HashAggregate (79) - : +- Exchange (78) - : +- HashAggregate (77) - : +- Scan parquet (76) - +- ShuffledHashJoin LeftSemi BuildRight (96) - :- Exchange (89) - : +- Filter (88) - : +- Scan parquet (87) - +- Project (95) - +- Filter (94) - +- HashAggregate (93) - +- Exchange (92) - +- HashAggregate (91) - +- Scan parquet (90) + TakeOrderedAndProject (102) + +- HashAggregate (101) + +- HashAggregate (100) + +- Project (99) + +- ShuffledHashJoin Inner BuildRight (98) + :- Exchange (87) + : +- Project (86) + : +- ShuffledHashJoin Inner BuildLeft (85) + : :- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (84) + : +- ShuffledHashJoin LeftSemi BuildRight (83) + : :- Exchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Project (82) + : +- Filter (81) + : +- HashAggregate (80) + : +- Exchange (79) + : +- HashAggregate (78) + : +- Scan parquet (77) + +- ShuffledHashJoin LeftSemi BuildRight (97) + :- Exchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- Project (96) + +- Filter (95) + +- HashAggregate (94) + +- Exchange (93) + +- HashAggregate (92) + +- Scan parquet (91) (1) Scan parquet @@ -186,372 +187,376 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(28) FilterExecTransformer +(29) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(29) ProjectExecTransformer +(30) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(30) ShuffledHashJoinExecTransformer +(31) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(33) VeloxAppendBatches +(34) VeloxAppendBatches Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(34) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(35) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(36) InputAdapter +(37) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(38) ShuffledHashJoinExecTransformer +(39) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(39) ProjectExecTransformer +(40) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(40) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(41) VeloxAppendBatches +(42) VeloxAppendBatches Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(42) ColumnarExchange +(43) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(43) ShuffleQueryStage +(44) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(44) InputAdapter +(45) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(45) InputIteratorTransformer +(46) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(46) Scan parquet +(47) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(47) FilterExecTransformer +(48) FilterExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Arguments: isnotnull(l_orderkey#X) -(48) ProjectExecTransformer +(49) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(49) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(50) VeloxAppendBatches +(51) VeloxAppendBatches Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: X -(51) ColumnarExchange +(52) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(53) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(53) InputAdapter +(54) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(54) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(55) ReusedExchange [Reuses operator id: 23] +(56) ReusedExchange [Reuses operator id: 24] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(56) ShuffleQueryStage +(57) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(57) InputAdapter +(58) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(58) InputIteratorTransformer +(59) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(59) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(60) FilterExecTransformer +(61) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(61) ProjectExecTransformer +(62) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(62) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(63) ShuffledHashJoinExecTransformer +(64) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(64) ProjectExecTransformer +(65) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(65) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(66) RegularHashAggregateExecTransformer +(67) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(67) WholeStageCodegenTransformer (X) +(68) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(68) TakeOrderedAndProjectExecTransformer +(69) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(69) VeloxColumnarToRowExec +(70) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(70) Scan parquet +(71) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(71) Filter +(72) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(72) Exchange +(73) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(73) Scan parquet +(74) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(75) Exchange +(76) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) Scan parquet +(77) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(77) HashAggregate +(78) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(78) Exchange +(79) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) HashAggregate +(80) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(80) Filter +(81) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(81) Project +(82) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(82) ShuffledHashJoin +(83) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(83) Exchange +(84) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) ShuffledHashJoin +(85) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(85) Project +(86) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(86) Exchange +(87) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Scan parquet +(88) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(88) Filter +(89) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(89) Exchange +(90) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(91) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(91) HashAggregate +(92) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(92) Exchange +(93) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) HashAggregate +(94) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(94) Filter +(95) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(95) Project +(96) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(96) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(97) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(98) Project +(99) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(99) HashAggregate +(100) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(100) HashAggregate +(101) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(101) TakeOrderedAndProject +(102) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(102) AdaptiveSparkPlan +(103) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt index ede94728ceb2..7b8c173fc086 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt @@ -1,112 +1,113 @@ == Physical Plan == -AdaptiveSparkPlan (127) +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ RegularHashAggregateExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ ProjectExecTransformer (82) - +- ^ FlushableHashAggregateExecTransformer (81) - +- ^ ProjectExecTransformer (80) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) - :- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67), Statistics(X) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50), Statistics(X) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) - : : :- ^ InputIteratorTransformer (9) - : : : +- ShuffleQueryStage (7), Statistics(X) - : : : +- ColumnarExchange (6) - : : : +- VeloxAppendBatches (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (44) - : : +- ShuffleQueryStage (42), Statistics(X) - : : +- ColumnarExchange (41) - : : +- VeloxAppendBatches (40) - : : +- ^ ProjectExecTransformer (38) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) - : : : :- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ FilterExecTransformer (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (26) - : : : +- ShuffleQueryStage (24), Statistics(X) - : : : +- ColumnarExchange (23) - : : : +- VeloxAppendBatches (22) - : : : +- ^ ProjectExecTransformer (20) - : : : +- ^ Scan parquet (19) - : : +- ^ InputIteratorTransformer (36) - : : +- ShuffleQueryStage (34), Statistics(X) - : : +- ColumnarExchange (33) - : : +- VeloxAppendBatches (32) - : : +- ^ ProjectExecTransformer (30) - : : +- ^ FilterExecTransformer (29) - : : +- ^ Scan parquet (28) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- VeloxAppendBatches (57) - : +- ^ ProjectExecTransformer (55) - : +- ^ FilterExecTransformer (54) - : +- ^ Scan parquet (53) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- VeloxAppendBatches (74) - +- ^ ProjectExecTransformer (72) - +- ^ FilterExecTransformer (71) - +- ^ Scan parquet (70) + VeloxColumnarToRowExec (91) + +- ^ RegularHashAggregateExecTransformer (89) + +- ^ InputIteratorTransformer (88) + +- ShuffleQueryStage (86), Statistics(X) + +- ColumnarExchange (85) + +- VeloxAppendBatches (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24), Statistics(X) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34), Statistics(X) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ FilterExecTransformer (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ FilterExecTransformer (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ FilterExecTransformer (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (116) - : +- Project (115) - : +- ShuffledHashJoin Inner BuildRight (114) - : :- Exchange (109) - : : +- Project (108) - : : +- ShuffledHashJoin Inner BuildLeft (107) - : : :- Exchange (93) - : : : +- Filter (92) - : : : +- Scan parquet (91) - : : +- Exchange (106) - : : +- ShuffledHashJoin LeftAnti BuildRight (105) - : : :- ShuffledHashJoin LeftSemi BuildRight (100) - : : : :- Exchange (97) - : : : : +- Project (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (99) - : : : +- Scan parquet (98) - : : +- Exchange (104) - : : +- Project (103) - : : +- Filter (102) - : : +- Scan parquet (101) - : +- Exchange (113) - : +- Project (112) - : +- Filter (111) - : +- Scan parquet (110) - +- Exchange (120) - +- Project (119) - +- Filter (118) - +- Scan parquet (117) + TakeOrderedAndProject (127) + +- HashAggregate (126) + +- Exchange (125) + +- HashAggregate (124) + +- Project (123) + +- ShuffledHashJoin Inner BuildRight (122) + :- Exchange (117) + : +- Project (116) + : +- ShuffledHashJoin Inner BuildRight (115) + : :- Exchange (110) + : : +- Project (109) + : : +- ShuffledHashJoin Inner BuildLeft (108) + : : :- Exchange (94) + : : : +- Filter (93) + : : : +- Scan parquet (92) + : : +- Exchange (107) + : : +- ShuffledHashJoin LeftAnti BuildRight (106) + : : :- ShuffledHashJoin LeftSemi BuildRight (101) + : : : :- Exchange (98) + : : : : +- Project (97) + : : : : +- Filter (96) + : : : : +- Scan parquet (95) + : : : +- Exchange (100) + : : : +- Scan parquet (99) + : : +- Exchange (105) + : : +- Project (104) + : : +- Filter (103) + : : +- Scan parquet (102) + : +- Exchange (114) + : +- Project (113) + : +- Filter (112) + : +- Scan parquet (111) + +- Exchange (121) + +- Project (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -453,211 +454,215 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(84) ColumnarExchange +(84) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(85) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(86) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(86) InputAdapter +(87) InputAdapter Input [2]: [s_name#X, count#X] -(87) InputIteratorTransformer +(88) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(88) RegularHashAggregateExecTransformer +(89) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(89) WholeStageCodegenTransformer (X) +(90) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(90) VeloxColumnarToRowExec +(91) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(91) Scan parquet +(92) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(93) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(94) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(95) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(96) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Project +(97) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(97) Exchange +(98) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(99) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(99) Exchange +(100) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) ShuffledHashJoin +(101) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(101) Scan parquet +(102) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(102) Filter +(103) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(103) Project +(104) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(104) Exchange +(105) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) ShuffledHashJoin +(106) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(106) Exchange +(107) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(108) Project +(109) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(109) Exchange +(110) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(112) Project +(113) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(113) Exchange +(114) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(115) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(115) Project +(116) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(116) Exchange +(117) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(118) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(118) Filter +(119) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(119) Project +(120) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(120) Exchange +(121) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(122) Project +(123) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(123) HashAggregate +(124) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(124) Exchange +(125) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(126) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(126) TakeOrderedAndProject +(127) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(127) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt index 58bbd22fa13c..d6ec93a97fc6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt @@ -1,46 +1,47 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (50) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- VeloxAppendBatches (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7), Statistics(X) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- VeloxAppendBatches (13) - +- ^ ProjectExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin LeftAnti BuildRight (42) - :- Exchange (39) - : +- Filter (38) - : +- Scan parquet (37) - +- Exchange (41) - +- Scan parquet (40) + Sort (49) + +- Exchange (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftAnti BuildRight (43) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (42) + +- Scan parquet (41) (1) Scan parquet @@ -137,255 +138,265 @@ Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(29) VeloxAppendBatches +(30) VeloxAppendBatches Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(30) ColumnarExchange +(31) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(32) InputAdapter +(33) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(33) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) SortExecTransformer +(35) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(35) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(36) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(37) Scan parquet +(38) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(38) Filter +(39) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(39) Exchange +(40) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) Scan parquet +(41) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(41) Exchange +(42) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(43) Project +(44) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(45) Exchange +(46) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(47) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(47) Exchange +(48) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(49) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(50) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (68) +AdaptiveSparkPlan (70) +- == Final Plan == - VeloxColumnarToRowExec (61) - +- ^ RegularHashAggregateExecTransformer (59) - +- ^ InputIteratorTransformer (58) - +- ShuffleQueryStage (56), Statistics(X) - +- ColumnarExchange (55) - +- ^ FlushableHashAggregateExecTransformer (53) - +- ^ ProjectExecTransformer (52) - +- ^ FilterExecTransformer (51) - +- ^ Scan parquet (50) + VeloxColumnarToRowExec (63) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ InputIteratorTransformer (60) + +- ShuffleQueryStage (58), Statistics(X) + +- ColumnarExchange (57) + +- VeloxAppendBatches (56) + +- ^ FlushableHashAggregateExecTransformer (54) + +- ^ ProjectExecTransformer (53) + +- ^ FilterExecTransformer (52) + +- ^ Scan parquet (51) +- == Initial Plan == - HashAggregate (67) - +- Exchange (66) - +- HashAggregate (65) - +- Project (64) - +- Filter (63) - +- Scan parquet (62) + HashAggregate (69) + +- Exchange (68) + +- HashAggregate (67) + +- Project (66) + +- Filter (65) + +- Scan parquet (64) -(50) Scan parquet +(51) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(51) FilterExecTransformer +(52) FilterExecTransformer Input [2]: [c_phone#X, c_acctbal#X] Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(52) ProjectExecTransformer +(53) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(53) FlushableHashAggregateExecTransformer +(54) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(54) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(55) ColumnarExchange +(56) VeloxAppendBatches +Input [2]: [sum#X, count#X] +Arguments: X + +(57) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(56) ShuffleQueryStage +(58) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(57) InputAdapter +(59) InputAdapter Input [2]: [sum#X, count#X] -(58) InputIteratorTransformer +(60) InputIteratorTransformer Input [2]: [sum#X, count#X] -(59) RegularHashAggregateExecTransformer +(61) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(60) WholeStageCodegenTransformer (X) +(62) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(61) VeloxColumnarToRowExec +(63) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(62) Scan parquet +(64) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(63) Filter +(65) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(64) Project +(66) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(65) HashAggregate +(67) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(66) Exchange +(68) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(67) HashAggregate +(69) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(68) AdaptiveSparkPlan +(70) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (68) +AdaptiveSparkPlan (70) +- == Final Plan == - VeloxColumnarToRowExec (61) - +- ^ RegularHashAggregateExecTransformer (59) - +- ^ InputIteratorTransformer (58) - +- ShuffleQueryStage (56), Statistics(X) - +- ColumnarExchange (55) - +- ^ FlushableHashAggregateExecTransformer (53) - +- ^ ProjectExecTransformer (52) - +- ^ FilterExecTransformer (51) - +- ^ Scan parquet (50) + VeloxColumnarToRowExec (63) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ InputIteratorTransformer (60) + +- ShuffleQueryStage (58), Statistics(X) + +- ColumnarExchange (57) + +- VeloxAppendBatches (56) + +- ^ FlushableHashAggregateExecTransformer (54) + +- ^ ProjectExecTransformer (53) + +- ^ FilterExecTransformer (52) + +- ^ Scan parquet (51) +- == Initial Plan == - HashAggregate (67) - +- Exchange (66) - +- HashAggregate (65) - +- Project (64) - +- Filter (63) - +- Scan parquet (62) \ No newline at end of file + HashAggregate (69) + +- Exchange (68) + +- HashAggregate (67) + +- Project (66) + +- Filter (65) + +- Scan parquet (64) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt index ebf46078f68b..a82dbf288086 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt @@ -1,50 +1,51 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32), Statistics(X) - +- ColumnarExchange (31) - +- VeloxAppendBatches (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7), Statistics(X) - : +- ColumnarExchange (6) - : +- VeloxAppendBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- VeloxAppendBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ FilterExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (52) - +- Exchange (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- ShuffledHashJoin LeftSemi BuildRight (46) - :- Exchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (53) + +- Exchange (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftSemi BuildRight (47) + :- Exchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -146,134 +147,138 @@ Input [2]: [o_orderpriority#X, count#X] Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(24) ColumnarExchange +(24) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(26) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(27) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(28) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(29) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(30) VeloxAppendBatches +(31) VeloxAppendBatches Input [2]: [o_orderpriority#X, order_count#X] Arguments: X -(31) ColumnarExchange +(32) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(35) SortExecTransformer +(36) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(37) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(41) Exchange +(42) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Scan parquet +(43) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(43) Filter +(44) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(44) Project +(45) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(45) Exchange +(46) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(47) Project +(48) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(48) HashAggregate +(49) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(49) Exchange +(50) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(51) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(51) Exchange +(52) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(53) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt index 8c3258e616f5..f20a52b91a8f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt @@ -1,126 +1,127 @@ == Physical Plan == -AdaptiveSparkPlan (145) +AdaptiveSparkPlan (146) +- == Final Plan == - VeloxColumnarToRowExec (105) - +- ^ SortExecTransformer (103) - +- ^ InputIteratorTransformer (102) - +- ShuffleQueryStage (100), Statistics(X) - +- ColumnarExchange (99) - +- VeloxAppendBatches (98) - +- ^ RegularHashAggregateExecTransformer (96) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ColumnarExchange (92) - +- ^ ProjectExecTransformer (90) - +- ^ FlushableHashAggregateExecTransformer (89) - +- ^ ProjectExecTransformer (88) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - :- ^ InputIteratorTransformer (77) - : +- ShuffleQueryStage (75), Statistics(X) - : +- ColumnarExchange (74) - : +- VeloxAppendBatches (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - : :- ^ InputIteratorTransformer (60) - : : +- ShuffleQueryStage (58), Statistics(X) - : : +- ColumnarExchange (57) - : : +- VeloxAppendBatches (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - : : :- ^ InputIteratorTransformer (43) - : : : +- ShuffleQueryStage (41), Statistics(X) - : : : +- ColumnarExchange (40) - : : : +- VeloxAppendBatches (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - : : : :- ^ InputIteratorTransformer (26) - : : : : +- ShuffleQueryStage (24), Statistics(X) - : : : : +- ColumnarExchange (23) - : : : : +- VeloxAppendBatches (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ InputIteratorTransformer (9) - : : : : : +- ShuffleQueryStage (7), Statistics(X) - : : : : : +- ColumnarExchange (6) - : : : : : +- VeloxAppendBatches (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ FilterExecTransformer (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (35) - : : : +- ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- VeloxAppendBatches (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ FilterExecTransformer (28) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50), Statistics(X) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ FilterExecTransformer (45) - : : +- ^ Scan parquet (44) - : +- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67), Statistics(X) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ FilterExecTransformer (62) - : +- ^ Scan parquet (61) - +- ^ InputIteratorTransformer (86) - +- ShuffleQueryStage (84), Statistics(X) - +- ColumnarExchange (83) - +- VeloxAppendBatches (82) - +- ^ ProjectExecTransformer (80) - +- ^ FilterExecTransformer (79) - +- ^ Scan parquet (78) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94), Statistics(X) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (144) - +- Exchange (143) - +- HashAggregate (142) - +- Exchange (141) - +- HashAggregate (140) - +- Project (139) - +- ShuffledHashJoin Inner BuildRight (138) - :- Exchange (133) - : +- Project (132) - : +- ShuffledHashJoin Inner BuildRight (131) - : :- Exchange (127) - : : +- Project (126) - : : +- ShuffledHashJoin Inner BuildRight (125) - : : :- Exchange (121) - : : : +- Project (120) - : : : +- ShuffledHashJoin Inner BuildRight (119) - : : : :- Exchange (115) - : : : : +- Project (114) - : : : : +- ShuffledHashJoin Inner BuildLeft (113) - : : : : :- Exchange (108) - : : : : : +- Filter (107) - : : : : : +- Scan parquet (106) - : : : : +- Exchange (112) - : : : : +- Project (111) - : : : : +- Filter (110) - : : : : +- Scan parquet (109) - : : : +- Exchange (118) - : : : +- Filter (117) - : : : +- Scan parquet (116) - : : +- Exchange (124) - : : +- Filter (123) - : : +- Scan parquet (122) - : +- Exchange (130) - : +- Filter (129) - : +- Scan parquet (128) - +- Exchange (137) - +- Project (136) - +- Filter (135) - +- Scan parquet (134) + Sort (145) + +- Exchange (144) + +- HashAggregate (143) + +- Exchange (142) + +- HashAggregate (141) + +- Project (140) + +- ShuffledHashJoin Inner BuildRight (139) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Project (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (138) + +- Project (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -498,250 +499,254 @@ Input [3]: [n_name#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(92) ColumnarExchange +(92) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(93) ShuffleQueryStage +(94) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(94) InputAdapter +(95) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(95) InputIteratorTransformer +(96) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(96) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(97) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(98) VeloxAppendBatches +(99) VeloxAppendBatches Input [2]: [n_name#X, revenue#X] Arguments: X -(99) ColumnarExchange +(100) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(100) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(101) InputAdapter +(102) InputAdapter Input [2]: [n_name#X, revenue#X] -(102) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(103) SortExecTransformer +(104) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(104) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(105) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(106) Scan parquet +(107) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(108) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(109) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Scan parquet +(110) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(110) Filter +(111) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(111) Project +(112) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(112) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(114) Project +(115) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(115) Exchange +(116) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) Scan parquet +(117) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(117) Filter +(118) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(118) Exchange +(119) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(120) Project +(121) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(121) Exchange +(122) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(123) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(123) Filter +(124) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(124) Exchange +(125) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(126) Project +(127) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(127) Exchange +(128) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(129) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(129) Filter +(130) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(130) Exchange +(131) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(132) Project +(133) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(133) Exchange +(134) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(134) Scan parquet +(135) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(135) Filter +(136) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(136) Project +(137) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(137) Exchange +(138) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(138) ShuffledHashJoin +(139) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(139) Project +(140) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(140) HashAggregate +(141) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(141) Exchange +(142) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) HashAggregate +(143) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(143) Exchange +(144) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(144) Sort +(145) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(145) AdaptiveSparkPlan +(146) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/6.txt index 415d85f46ee5..ddc921e22d0f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/6.txt @@ -7,7 +7,7 @@ AdaptiveSparkPlan (20) +- ShuffleQueryStage (8), Statistics(X) +- ColumnarExchange (7) +- VeloxAppendBatches (6) - +- ^ RegularHashAggregateExecTransformer (4) + +- ^ FlushableHashAggregateExecTransformer (4) +- ^ ProjectExecTransformer (3) +- ^ FilterExecTransformer (2) +- ^ Scan parquet (1) @@ -35,7 +35,7 @@ Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnot Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * l_discount#X) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) RegularHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt index fd60cdc1f478..710c6f3ba189 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt @@ -1,120 +1,121 @@ == Physical Plan == -AdaptiveSparkPlan (138) +AdaptiveSparkPlan (139) +- == Final Plan == - VeloxColumnarToRowExec (100) - +- ^ SortExecTransformer (98) - +- ^ InputIteratorTransformer (97) - +- ShuffleQueryStage (95), Statistics(X) - +- ColumnarExchange (94) - +- VeloxAppendBatches (93) - +- ^ RegularHashAggregateExecTransformer (91) - +- ^ InputIteratorTransformer (90) - +- ShuffleQueryStage (88), Statistics(X) - +- ColumnarExchange (87) - +- ^ ProjectExecTransformer (85) - +- ^ FlushableHashAggregateExecTransformer (84) - +- ^ ProjectExecTransformer (83) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) - :- ^ InputIteratorTransformer (77) - : +- ShuffleQueryStage (75), Statistics(X) - : +- ColumnarExchange (74) - : +- VeloxAppendBatches (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - : :- ^ InputIteratorTransformer (60) - : : +- ShuffleQueryStage (58), Statistics(X) - : : +- ColumnarExchange (57) - : : +- VeloxAppendBatches (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - : : :- ^ InputIteratorTransformer (43) - : : : +- ShuffleQueryStage (41), Statistics(X) - : : : +- ColumnarExchange (40) - : : : +- VeloxAppendBatches (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : :- ^ InputIteratorTransformer (26) - : : : : +- ShuffleQueryStage (24), Statistics(X) - : : : : +- ColumnarExchange (23) - : : : : +- VeloxAppendBatches (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ InputIteratorTransformer (9) - : : : : : +- ShuffleQueryStage (7), Statistics(X) - : : : : : +- ColumnarExchange (6) - : : : : : +- VeloxAppendBatches (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ FilterExecTransformer (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (35) - : : : +- ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- VeloxAppendBatches (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ FilterExecTransformer (28) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50), Statistics(X) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ FilterExecTransformer (45) - : : +- ^ Scan parquet (44) - : +- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67), Statistics(X) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ FilterExecTransformer (62) - : +- ^ Scan parquet (61) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ReusedExchange (78) + VeloxColumnarToRowExec (101) + +- ^ SortExecTransformer (99) + +- ^ InputIteratorTransformer (98) + +- ShuffleQueryStage (96), Statistics(X) + +- ColumnarExchange (95) + +- VeloxAppendBatches (94) + +- ^ RegularHashAggregateExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- VeloxAppendBatches (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ReusedExchange (78) +- == Initial Plan == - Sort (137) - +- Exchange (136) - +- HashAggregate (135) - +- Exchange (134) - +- HashAggregate (133) - +- Project (132) - +- ShuffledHashJoin Inner BuildRight (131) - :- Exchange (127) - : +- Project (126) - : +- ShuffledHashJoin Inner BuildRight (125) - : :- Exchange (121) - : : +- Project (120) - : : +- ShuffledHashJoin Inner BuildRight (119) - : : :- Exchange (115) - : : : +- Project (114) - : : : +- ShuffledHashJoin Inner BuildRight (113) - : : : :- Exchange (109) - : : : : +- Project (108) - : : : : +- ShuffledHashJoin Inner BuildLeft (107) - : : : : :- Exchange (103) - : : : : : +- Filter (102) - : : : : : +- Scan parquet (101) - : : : : +- Exchange (106) - : : : : +- Filter (105) - : : : : +- Scan parquet (104) - : : : +- Exchange (112) - : : : +- Filter (111) - : : : +- Scan parquet (110) - : : +- Exchange (118) - : : +- Filter (117) - : : +- Scan parquet (116) - : +- Exchange (124) - : +- Filter (123) - : +- Scan parquet (122) - +- Exchange (130) - +- Filter (129) - +- Scan parquet (128) + Sort (138) + +- Exchange (137) + +- HashAggregate (136) + +- Exchange (135) + +- HashAggregate (134) + +- Project (133) + +- ShuffledHashJoin Inner BuildRight (132) + :- Exchange (128) + : +- Project (127) + : +- ShuffledHashJoin Inner BuildRight (126) + : :- Exchange (122) + : : +- Project (121) + : : +- ShuffledHashJoin Inner BuildRight (120) + : : :- Exchange (116) + : : : +- Project (115) + : : : +- ShuffledHashJoin Inner BuildRight (114) + : : : :- Exchange (110) + : : : : +- Project (109) + : : : : +- ShuffledHashJoin Inner BuildLeft (108) + : : : : :- Exchange (104) + : : : : : +- Filter (103) + : : : : : +- Scan parquet (102) + : : : : +- Exchange (107) + : : : : +- Filter (106) + : : : : +- Scan parquet (105) + : : : +- Exchange (113) + : : : +- Filter (112) + : : : +- Scan parquet (111) + : : +- Exchange (119) + : : +- Filter (118) + : : +- Scan parquet (117) + : +- Exchange (125) + : +- Filter (124) + : +- Scan parquet (123) + +- Exchange (131) + +- Filter (130) + +- Scan parquet (129) (1) Scan parquet @@ -468,242 +469,246 @@ Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(87) ColumnarExchange +(87) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(88) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(88) ShuffleQueryStage +(89) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(89) InputAdapter +(90) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(90) InputIteratorTransformer +(91) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(91) RegularHashAggregateExecTransformer +(92) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(92) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(93) VeloxAppendBatches +(94) VeloxAppendBatches Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(94) ColumnarExchange +(95) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(95) ShuffleQueryStage +(96) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(96) InputAdapter +(97) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(97) InputIteratorTransformer +(98) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(98) SortExecTransformer +(99) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(99) WholeStageCodegenTransformer (X) +(100) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(100) VeloxColumnarToRowExec +(101) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(101) Scan parquet +(102) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(102) Filter +(103) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(103) Exchange +(104) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) Scan parquet +(105) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(105) Filter +(106) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(106) Exchange +(107) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(108) Project +(109) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(109) Exchange +(110) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(112) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(114) Project +(115) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(115) Exchange +(116) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) Scan parquet +(117) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(117) Filter +(118) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(118) Exchange +(119) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(120) Project +(121) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(121) Exchange +(122) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(123) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(123) Filter +(124) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(124) Exchange +(125) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(126) Project +(127) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(127) Exchange +(128) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(129) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(129) Filter +(130) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(130) Exchange +(131) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(132) Project +(133) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(133) HashAggregate +(134) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(134) Exchange +(135) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(135) HashAggregate +(136) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(136) Exchange +(137) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) Sort +(138) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(138) AdaptiveSparkPlan +(139) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt index 5427956bb9d1..953a3a8f0a7c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt @@ -1,165 +1,166 @@ == Physical Plan == -AdaptiveSparkPlan (192) +AdaptiveSparkPlan (193) +- == Final Plan == - VeloxColumnarToRowExec (140) - +- ^ SortExecTransformer (138) - +- ^ InputIteratorTransformer (137) - +- ShuffleQueryStage (135), Statistics(X) - +- ColumnarExchange (134) - +- VeloxAppendBatches (133) - +- ^ ProjectExecTransformer (131) - +- ^ RegularHashAggregateExecTransformer (130) - +- ^ InputIteratorTransformer (129) - +- ShuffleQueryStage (127), Statistics(X) - +- ColumnarExchange (126) - +- ^ ProjectExecTransformer (124) - +- ^ FlushableHashAggregateExecTransformer (123) - +- ^ ProjectExecTransformer (122) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) - :- ^ InputIteratorTransformer (111) - : +- ShuffleQueryStage (109), Statistics(X) - : +- ColumnarExchange (108) - : +- VeloxAppendBatches (107) - : +- ^ ProjectExecTransformer (105) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) - : :- ^ InputIteratorTransformer (94) - : : +- ShuffleQueryStage (92), Statistics(X) - : : +- ColumnarExchange (91) - : : +- VeloxAppendBatches (90) - : : +- ^ ProjectExecTransformer (88) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - : : :- ^ InputIteratorTransformer (77) - : : : +- ShuffleQueryStage (75), Statistics(X) - : : : +- ColumnarExchange (74) - : : : +- VeloxAppendBatches (73) - : : : +- ^ ProjectExecTransformer (71) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) - : : : :- ^ InputIteratorTransformer (60) - : : : : +- ShuffleQueryStage (58), Statistics(X) - : : : : +- ColumnarExchange (57) - : : : : +- VeloxAppendBatches (56) - : : : : +- ^ ProjectExecTransformer (54) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) - : : : : :- ^ InputIteratorTransformer (43) - : : : : : +- ShuffleQueryStage (41), Statistics(X) - : : : : : +- ColumnarExchange (40) - : : : : : +- VeloxAppendBatches (39) - : : : : : +- ^ ProjectExecTransformer (37) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : : : :- ^ InputIteratorTransformer (26) - : : : : : : +- ShuffleQueryStage (24), Statistics(X) - : : : : : : +- ColumnarExchange (23) - : : : : : : +- VeloxAppendBatches (22) - : : : : : : +- ^ ProjectExecTransformer (20) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : : : :- ^ InputIteratorTransformer (9) - : : : : : : : +- ShuffleQueryStage (7), Statistics(X) - : : : : : : : +- ColumnarExchange (6) - : : : : : : : +- VeloxAppendBatches (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (18) - : : : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : : : +- ColumnarExchange (15) - : : : : : : +- VeloxAppendBatches (14) - : : : : : : +- ^ ProjectExecTransformer (12) - : : : : : : +- ^ FilterExecTransformer (11) - : : : : : : +- ^ Scan parquet (10) - : : : : : +- ^ InputIteratorTransformer (35) - : : : : : +- ShuffleQueryStage (33), Statistics(X) - : : : : : +- ColumnarExchange (32) - : : : : : +- VeloxAppendBatches (31) - : : : : : +- ^ ProjectExecTransformer (29) - : : : : : +- ^ FilterExecTransformer (28) - : : : : : +- ^ Scan parquet (27) - : : : : +- ^ InputIteratorTransformer (52) - : : : : +- ShuffleQueryStage (50), Statistics(X) - : : : : +- ColumnarExchange (49) - : : : : +- VeloxAppendBatches (48) - : : : : +- ^ ProjectExecTransformer (46) - : : : : +- ^ FilterExecTransformer (45) - : : : : +- ^ Scan parquet (44) - : : : +- ^ InputIteratorTransformer (69) - : : : +- ShuffleQueryStage (67), Statistics(X) - : : : +- ColumnarExchange (66) - : : : +- VeloxAppendBatches (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ FilterExecTransformer (62) - : : : +- ^ Scan parquet (61) - : : +- ^ InputIteratorTransformer (86) - : : +- ShuffleQueryStage (84), Statistics(X) - : : +- ColumnarExchange (83) - : : +- VeloxAppendBatches (82) - : : +- ^ ProjectExecTransformer (80) - : : +- ^ FilterExecTransformer (79) - : : +- ^ Scan parquet (78) - : +- ^ InputIteratorTransformer (103) - : +- ShuffleQueryStage (101), Statistics(X) - : +- ColumnarExchange (100) - : +- VeloxAppendBatches (99) - : +- ^ ProjectExecTransformer (97) - : +- ^ FilterExecTransformer (96) - : +- ^ Scan parquet (95) - +- ^ InputIteratorTransformer (120) - +- ShuffleQueryStage (118), Statistics(X) - +- ColumnarExchange (117) - +- VeloxAppendBatches (116) - +- ^ ProjectExecTransformer (114) - +- ^ FilterExecTransformer (113) - +- ^ Scan parquet (112) + VeloxColumnarToRowExec (141) + +- ^ SortExecTransformer (139) + +- ^ InputIteratorTransformer (138) + +- ShuffleQueryStage (136), Statistics(X) + +- ColumnarExchange (135) + +- VeloxAppendBatches (134) + +- ^ ProjectExecTransformer (132) + +- ^ RegularHashAggregateExecTransformer (131) + +- ^ InputIteratorTransformer (130) + +- ShuffleQueryStage (128), Statistics(X) + +- ColumnarExchange (127) + +- VeloxAppendBatches (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109), Statistics(X) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92), Statistics(X) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75), Statistics(X) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58), Statistics(X) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41), Statistics(X) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ FilterExecTransformer (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33), Statistics(X) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ FilterExecTransformer (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50), Statistics(X) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ FilterExecTransformer (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67), Statistics(X) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ FilterExecTransformer (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ FilterExecTransformer (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ FilterExecTransformer (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118), Statistics(X) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ FilterExecTransformer (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (191) - +- Exchange (190) - +- HashAggregate (189) - +- Exchange (188) - +- HashAggregate (187) - +- Project (186) - +- ShuffledHashJoin Inner BuildRight (185) - :- Exchange (180) - : +- Project (179) - : +- ShuffledHashJoin Inner BuildRight (178) - : :- Exchange (174) - : : +- Project (173) - : : +- ShuffledHashJoin Inner BuildRight (172) - : : :- Exchange (168) - : : : +- Project (167) - : : : +- ShuffledHashJoin Inner BuildRight (166) - : : : :- Exchange (162) - : : : : +- Project (161) - : : : : +- ShuffledHashJoin Inner BuildRight (160) - : : : : :- Exchange (156) - : : : : : +- Project (155) - : : : : : +- ShuffledHashJoin Inner BuildRight (154) - : : : : : :- Exchange (150) - : : : : : : +- Project (149) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (148) - : : : : : : :- Exchange (144) - : : : : : : : +- Project (143) - : : : : : : : +- Filter (142) - : : : : : : : +- Scan parquet (141) - : : : : : : +- Exchange (147) - : : : : : : +- Filter (146) - : : : : : : +- Scan parquet (145) - : : : : : +- Exchange (153) - : : : : : +- Filter (152) - : : : : : +- Scan parquet (151) - : : : : +- Exchange (159) - : : : : +- Filter (158) - : : : : +- Scan parquet (157) - : : : +- Exchange (165) - : : : +- Filter (164) - : : : +- Scan parquet (163) - : : +- Exchange (171) - : : +- Filter (170) - : : +- Scan parquet (169) - : +- Exchange (177) - : +- Filter (176) - : +- Scan parquet (175) - +- Exchange (184) - +- Project (183) - +- Filter (182) - +- Scan parquet (181) + Sort (192) + +- Exchange (191) + +- HashAggregate (190) + +- Exchange (189) + +- HashAggregate (188) + +- Project (187) + +- ShuffledHashJoin Inner BuildRight (186) + :- Exchange (181) + : +- Project (180) + : +- ShuffledHashJoin Inner BuildRight (179) + : :- Exchange (175) + : : +- Project (174) + : : +- ShuffledHashJoin Inner BuildRight (173) + : : :- Exchange (169) + : : : +- Project (168) + : : : +- ShuffledHashJoin Inner BuildRight (167) + : : : :- Exchange (163) + : : : : +- Project (162) + : : : : +- ShuffledHashJoin Inner BuildRight (161) + : : : : :- Exchange (157) + : : : : : +- Project (156) + : : : : : +- ShuffledHashJoin Inner BuildRight (155) + : : : : : :- Exchange (151) + : : : : : : +- Project (150) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (149) + : : : : : : :- Exchange (145) + : : : : : : : +- Project (144) + : : : : : : : +- Filter (143) + : : : : : : : +- Scan parquet (142) + : : : : : : +- Exchange (148) + : : : : : : +- Filter (147) + : : : : : : +- Scan parquet (146) + : : : : : +- Exchange (154) + : : : : : +- Filter (153) + : : : : : +- Scan parquet (152) + : : : : +- Exchange (160) + : : : : +- Filter (159) + : : : : +- Scan parquet (158) + : : : +- Exchange (166) + : : : +- Filter (165) + : : : +- Scan parquet (164) + : : +- Exchange (172) + : : +- Filter (171) + : : +- Scan parquet (170) + : +- Exchange (178) + : +- Filter (177) + : +- Scan parquet (176) + +- Exchange (185) + +- Project (184) + +- Filter (183) + +- Scan parquet (182) (1) Scan parquet @@ -675,312 +676,316 @@ Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(126) ColumnarExchange +(126) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(127) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(127) ShuffleQueryStage +(128) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(128) InputAdapter +(129) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(129) InputIteratorTransformer +(130) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(130) RegularHashAggregateExecTransformer +(131) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(131) ProjectExecTransformer +(132) ProjectExecTransformer Output [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(132) WholeStageCodegenTransformer (X) +(133) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(133) VeloxAppendBatches +(134) VeloxAppendBatches Input [2]: [o_year#X, mkt_share#X] Arguments: X -(134) ColumnarExchange +(135) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(135) ShuffleQueryStage +(136) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(136) InputAdapter +(137) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(137) InputIteratorTransformer +(138) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(138) SortExecTransformer +(139) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(139) WholeStageCodegenTransformer (X) +(140) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(140) VeloxColumnarToRowExec +(141) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(141) Scan parquet +(142) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(142) Filter +(143) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(143) Project +(144) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(144) Exchange +(145) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) Scan parquet +(146) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(146) Filter +(147) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(147) Exchange +(148) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) ShuffledHashJoin +(149) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(149) Project +(150) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(150) Exchange +(151) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) Scan parquet +(152) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(152) Filter +(153) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(153) Exchange +(154) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) ShuffledHashJoin +(155) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(155) Project +(156) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(156) Exchange +(157) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) Scan parquet +(158) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(158) Filter +(159) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(159) Exchange +(160) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) ShuffledHashJoin +(161) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(161) Project +(162) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(162) Exchange +(163) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) Scan parquet +(164) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(164) Filter +(165) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(165) Exchange +(166) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) ShuffledHashJoin +(167) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(167) Project +(168) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(168) Exchange +(169) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(169) Scan parquet +(170) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(170) Filter +(171) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(171) Exchange +(172) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(172) ShuffledHashJoin +(173) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(173) Project +(174) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(174) Exchange +(175) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(175) Scan parquet +(176) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(176) Filter +(177) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(177) Exchange +(178) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(178) ShuffledHashJoin +(179) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(179) Project +(180) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(180) Exchange +(181) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(181) Scan parquet +(182) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(182) Filter +(183) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(183) Project +(184) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(184) Exchange +(185) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(185) ShuffledHashJoin +(186) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(186) Project +(187) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(187) HashAggregate +(188) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(188) Exchange +(189) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(189) HashAggregate +(190) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(190) Exchange +(191) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(191) Sort +(192) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(192) AdaptiveSparkPlan +(193) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt index af275ac9926c..492ff1aeadd0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt @@ -1,125 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (144) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (105) - +- ^ SortExecTransformer (103) - +- ^ InputIteratorTransformer (102) - +- ShuffleQueryStage (100), Statistics(X) - +- ColumnarExchange (99) - +- VeloxAppendBatches (98) - +- ^ RegularHashAggregateExecTransformer (96) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ColumnarExchange (92) - +- ^ ProjectExecTransformer (90) - +- ^ FlushableHashAggregateExecTransformer (89) - +- ^ ProjectExecTransformer (88) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - :- ^ InputIteratorTransformer (77) - : +- ShuffleQueryStage (75), Statistics(X) - : +- ColumnarExchange (74) - : +- VeloxAppendBatches (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) - : :- ^ InputIteratorTransformer (60) - : : +- ShuffleQueryStage (58), Statistics(X) - : : +- ColumnarExchange (57) - : : +- VeloxAppendBatches (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) - : : :- ^ InputIteratorTransformer (43) - : : : +- ShuffleQueryStage (41), Statistics(X) - : : : +- ColumnarExchange (40) - : : : +- VeloxAppendBatches (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : :- ^ InputIteratorTransformer (26) - : : : : +- ShuffleQueryStage (24), Statistics(X) - : : : : +- ColumnarExchange (23) - : : : : +- VeloxAppendBatches (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ InputIteratorTransformer (9) - : : : : : +- ShuffleQueryStage (7), Statistics(X) - : : : : : +- ColumnarExchange (6) - : : : : : +- VeloxAppendBatches (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxAppendBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ FilterExecTransformer (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ InputIteratorTransformer (35) - : : : +- ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- VeloxAppendBatches (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ FilterExecTransformer (28) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50), Statistics(X) - : : +- ColumnarExchange (49) - : : +- VeloxAppendBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ FilterExecTransformer (45) - : : +- ^ Scan parquet (44) - : +- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67), Statistics(X) - : +- ColumnarExchange (66) - : +- VeloxAppendBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ FilterExecTransformer (62) - : +- ^ Scan parquet (61) - +- ^ InputIteratorTransformer (86) - +- ShuffleQueryStage (84), Statistics(X) - +- ColumnarExchange (83) - +- VeloxAppendBatches (82) - +- ^ ProjectExecTransformer (80) - +- ^ FilterExecTransformer (79) - +- ^ Scan parquet (78) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94), Statistics(X) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (143) - +- Exchange (142) - +- HashAggregate (141) - +- Exchange (140) - +- HashAggregate (139) - +- Project (138) - +- ShuffledHashJoin Inner BuildRight (137) - :- Exchange (133) - : +- Project (132) - : +- ShuffledHashJoin Inner BuildRight (131) - : :- Exchange (127) - : : +- Project (126) - : : +- ShuffledHashJoin Inner BuildRight (125) - : : :- Exchange (121) - : : : +- Project (120) - : : : +- ShuffledHashJoin Inner BuildRight (119) - : : : :- Exchange (115) - : : : : +- Project (114) - : : : : +- ShuffledHashJoin Inner BuildLeft (113) - : : : : :- Exchange (109) - : : : : : +- Project (108) - : : : : : +- Filter (107) - : : : : : +- Scan parquet (106) - : : : : +- Exchange (112) - : : : : +- Filter (111) - : : : : +- Scan parquet (110) - : : : +- Exchange (118) - : : : +- Filter (117) - : : : +- Scan parquet (116) - : : +- Exchange (124) - : : +- Filter (123) - : : +- Scan parquet (122) - : +- Exchange (130) - : +- Filter (129) - : +- Scan parquet (128) - +- Exchange (136) - +- Filter (135) - +- Scan parquet (134) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (110) + : : : : : +- Project (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Filter (112) + : : : : +- Scan parquet (111) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -497,246 +498,250 @@ Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(92) ColumnarExchange +(92) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(93) ShuffleQueryStage +(94) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(94) InputAdapter +(95) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(95) InputIteratorTransformer +(96) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(96) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(97) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(98) VeloxAppendBatches +(99) VeloxAppendBatches Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(99) ColumnarExchange +(100) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(100) ShuffleQueryStage +(101) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(101) InputAdapter +(102) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(102) InputIteratorTransformer +(103) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(103) SortExecTransformer +(104) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(104) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(105) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(106) Scan parquet +(107) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(107) Filter +(108) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(108) Project +(109) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(109) Exchange +(110) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(111) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(112) Exchange +(113) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(114) Project +(115) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(115) Exchange +(116) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) Scan parquet +(117) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(117) Filter +(118) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(118) Exchange +(119) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(120) Project +(121) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(121) Exchange +(122) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(123) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(123) Filter +(124) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(124) Exchange +(125) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(126) Project +(127) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(127) Exchange +(128) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(129) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(129) Filter +(130) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(130) Exchange +(131) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(132) Project +(133) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(133) Exchange +(134) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(134) Scan parquet +(135) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(135) Filter +(136) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(136) Exchange +(137) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(138) Project +(139) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(139) HashAggregate +(140) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(140) Exchange +(141) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(141) HashAggregate +(142) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(142) Exchange +(143) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(143) Sort +(144) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(144) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file From 451663595a6c58ad235057768f5dabb9f3bedf13 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Fri, 14 Jun 2024 16:18:59 +0800 Subject: [PATCH 6/9] fixup --- .../apache/gluten/extension/FlushableHashAggregateRule.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala b/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala index 3910acbebb4c..854b4969b71d 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala @@ -96,7 +96,8 @@ object FlushableHashAggregateRule { * only on a single partition among the whole cluster. Spark's planner may use this information to * perform optimizations like doing "partial_count(a, b, c)" directly on the output data. */ - private def isAggInputAlreadyDistributedWithAggKeys(agg: HashAggregateExecTransformer): Boolean = { + private def isAggInputAlreadyDistributedWithAggKeys( + agg: HashAggregateExecTransformer): Boolean = { if (agg.groupingExpressions.isEmpty) { // Empty grouping set () should not be satisfied by any partitioning patterns. // E.g., From 7e3aca6135d058d8e7244491b479bb8875a08c32 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Fri, 14 Jun 2024 16:19:56 +0800 Subject: [PATCH 7/9] fixup --- .../apache/gluten/extension/FlushableHashAggregateRule.scala | 3 --- 1 file changed, 3 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala b/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala index 854b4969b71d..777bf553856e 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala @@ -72,9 +72,6 @@ case class FlushableHashAggregateRule(session: SparkSession) extends Rule[SparkP } val out = transformDown(plan) - if (out.fastEquals(plan)) { - return plan - } out } From c21fad265f94acce4132c7f88df2ac38bf051594 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Fri, 14 Jun 2024 21:07:03 +0800 Subject: [PATCH 8/9] fixup --- .../scala/org/apache/spark/sql/GlutenImplicitsTest.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenImplicitsTest.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenImplicitsTest.scala index e4356cec8ff1..9e120945be34 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenImplicitsTest.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenImplicitsTest.scala @@ -79,10 +79,10 @@ class GlutenImplicitsTest extends GlutenSQLTestsBaseTrait { testGluten("fallbackSummary with shuffle") { withAQEEnabledAndDisabled { val df = spark.sql("SELECT c2 FROM t1 group by c2").filter(_.getLong(0) > 0) - assert(df.fallbackSummary().numGlutenNodes == 5, df.fallbackSummary()) + assert(df.fallbackSummary().numGlutenNodes == 6, df.fallbackSummary()) assert(df.fallbackSummary().numFallbackNodes == 1, df.fallbackSummary()) df.collect() - assert(df.fallbackSummary().numGlutenNodes == 5, df.fallbackSummary()) + assert(df.fallbackSummary().numGlutenNodes == 6, df.fallbackSummary()) assert(df.fallbackSummary().numFallbackNodes == 1, df.fallbackSummary()) } } @@ -119,10 +119,10 @@ class GlutenImplicitsTest extends GlutenSQLTestsBaseTrait { testGluten("fallbackSummary with cached data and shuffle") { withAQEEnabledAndDisabled { val df = spark.sql("select * from t1").filter(_.getLong(0) > 0).cache.repartition() - assert(df.fallbackSummary().numGlutenNodes == 6, df.fallbackSummary()) + assert(df.fallbackSummary().numGlutenNodes == 7, df.fallbackSummary()) assert(df.fallbackSummary().numFallbackNodes == 1, df.fallbackSummary()) df.collect() - assert(df.fallbackSummary().numGlutenNodes == 6, df.fallbackSummary()) + assert(df.fallbackSummary().numGlutenNodes == 7, df.fallbackSummary()) assert(df.fallbackSummary().numFallbackNodes == 1, df.fallbackSummary()) } } From 46cde2c6aeb56cd1907b7b017b5595d9a4ec962f Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Mon, 17 Jun 2024 11:55:11 +0800 Subject: [PATCH 9/9] fixup --- .../sql/errors/GlutenQueryExecutionErrorsSuite.scala | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/errors/GlutenQueryExecutionErrorsSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/errors/GlutenQueryExecutionErrorsSuite.scala index 8896541c29d2..c25e65cf0b68 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/errors/GlutenQueryExecutionErrorsSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/errors/GlutenQueryExecutionErrorsSuite.scala @@ -16,11 +16,20 @@ */ package org.apache.spark.sql.errors +import org.apache.spark.SparkConf import org.apache.spark.sql.GlutenSQLTestsBaseTrait class GlutenQueryExecutionErrorsSuite extends QueryExecutionErrorsSuite with GlutenSQLTestsBaseTrait { + + override def sparkConf: SparkConf = { + // Disables VeloxAppendBatches in which GeneralOutIterator wraps vanilla Spark's exceptions + // with GlutenException. + super.sparkConf + .set("spark.gluten.sql.columnar.backend.velox.coalesceBatchesBeforeShuffle", "false") + } + override protected def getResourceParquetFilePath(name: String): String = { getWorkspaceFilePath("sql", "core", "src", "test", "resources").toString + "/" + name }